diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java index 448ce9f7b..d16d417b9 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java @@ -84,7 +84,7 @@ public class HoodieWriteConfig extends HoodieConfig { .noDefaultValue() .withDocumentation("Table name that will be used for registering with metastores like HMS. Needs to be same across runs."); - public static final ConfigProperty PRECOMBINE_FIELD = ConfigProperty + public static final ConfigProperty PRECOMBINE_FIELD_NAME = ConfigProperty .key("hoodie.datasource.write.precombine.field") .defaultValue("ts") .withDocumentation("Field used in preCombining before actual write. When two records have the same key value, " @@ -423,10 +423,10 @@ public class HoodieWriteConfig extends HoodieConfig { @Deprecated public static final String TABLE_NAME = TBL_NAME.key(); /** - * @deprecated Use {@link #PRECOMBINE_FIELD} and its methods instead + * @deprecated Use {@link #PRECOMBINE_FIELD_NAME} and its methods instead */ @Deprecated - public static final String PRECOMBINE_FIELD_PROP = PRECOMBINE_FIELD.key(); + public static final String PRECOMBINE_FIELD_PROP = PRECOMBINE_FIELD_NAME.key(); /** * @deprecated Use {@link #WRITE_PAYLOAD_CLASS_NAME} and its methods instead */ @@ -859,7 +859,7 @@ public class HoodieWriteConfig extends HoodieConfig { } public String getPreCombineField() { - return getString(PRECOMBINE_FIELD); + return getString(PRECOMBINE_FIELD_NAME); } public String getWritePayloadClass() { @@ -1803,7 +1803,7 @@ public class HoodieWriteConfig extends HoodieConfig { } public Builder withPreCombineField(String preCombineField) { - writeConfig.setValue(PRECOMBINE_FIELD, preCombineField); + writeConfig.setValue(PRECOMBINE_FIELD_NAME, preCombineField); return this; } diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/keygen/ComplexAvroKeyGenerator.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/keygen/ComplexAvroKeyGenerator.java index 87ddddd58..375225f48 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/keygen/ComplexAvroKeyGenerator.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/keygen/ComplexAvroKeyGenerator.java @@ -32,9 +32,9 @@ public class ComplexAvroKeyGenerator extends BaseKeyGenerator { public ComplexAvroKeyGenerator(TypedProperties props) { super(props); - this.recordKeyFields = Arrays.stream(props.getString(KeyGeneratorOptions.RECORDKEY_FIELD.key()) + this.recordKeyFields = Arrays.stream(props.getString(KeyGeneratorOptions.RECORDKEY_FIELD_NAME.key()) .split(",")).map(String::trim).filter(s -> !s.isEmpty()).collect(Collectors.toList()); - this.partitionPathFields = Arrays.stream(props.getString(KeyGeneratorOptions.PARTITIONPATH_FIELD.key()) + this.partitionPathFields = Arrays.stream(props.getString(KeyGeneratorOptions.PARTITIONPATH_FIELD_NAME.key()) .split(",")).map(String::trim).filter(s -> !s.isEmpty()).collect(Collectors.toList()); } diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/keygen/CustomAvroKeyGenerator.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/keygen/CustomAvroKeyGenerator.java index 286e495c2..370b57b53 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/keygen/CustomAvroKeyGenerator.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/keygen/CustomAvroKeyGenerator.java @@ -55,8 +55,8 @@ public class CustomAvroKeyGenerator extends BaseKeyGenerator { public CustomAvroKeyGenerator(TypedProperties props) { super(props); - this.recordKeyFields = Arrays.stream(props.getString(KeyGeneratorOptions.RECORDKEY_FIELD.key()).split(",")).map(String::trim).collect(Collectors.toList()); - this.partitionPathFields = Arrays.stream(props.getString(KeyGeneratorOptions.PARTITIONPATH_FIELD.key()).split(",")).map(String::trim).collect(Collectors.toList()); + this.recordKeyFields = Arrays.stream(props.getString(KeyGeneratorOptions.RECORDKEY_FIELD_NAME.key()).split(",")).map(String::trim).collect(Collectors.toList()); + this.partitionPathFields = Arrays.stream(props.getString(KeyGeneratorOptions.PARTITIONPATH_FIELD_NAME.key()).split(",")).map(String::trim).collect(Collectors.toList()); } @Override diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/keygen/GlobalAvroDeleteKeyGenerator.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/keygen/GlobalAvroDeleteKeyGenerator.java index 3b20c8a8f..674e28647 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/keygen/GlobalAvroDeleteKeyGenerator.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/keygen/GlobalAvroDeleteKeyGenerator.java @@ -35,7 +35,7 @@ public class GlobalAvroDeleteKeyGenerator extends BaseKeyGenerator { public GlobalAvroDeleteKeyGenerator(TypedProperties config) { super(config); - this.recordKeyFields = Arrays.asList(config.getString(KeyGeneratorOptions.RECORDKEY_FIELD.key()).split(",")); + this.recordKeyFields = Arrays.asList(config.getString(KeyGeneratorOptions.RECORDKEY_FIELD_NAME.key()).split(",")); } @Override diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/keygen/NonpartitionedAvroKeyGenerator.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/keygen/NonpartitionedAvroKeyGenerator.java index b1e0ca7a5..4384f171d 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/keygen/NonpartitionedAvroKeyGenerator.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/keygen/NonpartitionedAvroKeyGenerator.java @@ -36,7 +36,7 @@ public class NonpartitionedAvroKeyGenerator extends BaseKeyGenerator { public NonpartitionedAvroKeyGenerator(TypedProperties props) { super(props); - this.recordKeyFields = Arrays.stream(props.getString(KeyGeneratorOptions.RECORDKEY_FIELD.key()) + this.recordKeyFields = Arrays.stream(props.getString(KeyGeneratorOptions.RECORDKEY_FIELD_NAME.key()) .split(",")).map(String::trim).filter(s -> !s.isEmpty()).collect(Collectors.toList()); this.partitionPathFields = EMPTY_PARTITION_FIELD_LIST; } diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/keygen/SimpleAvroKeyGenerator.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/keygen/SimpleAvroKeyGenerator.java index 2aa939207..dae55c5f8 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/keygen/SimpleAvroKeyGenerator.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/keygen/SimpleAvroKeyGenerator.java @@ -29,8 +29,8 @@ import java.util.Collections; public class SimpleAvroKeyGenerator extends BaseKeyGenerator { public SimpleAvroKeyGenerator(TypedProperties props) { - this(props, props.getString(KeyGeneratorOptions.RECORDKEY_FIELD.key()), - props.getString(KeyGeneratorOptions.PARTITIONPATH_FIELD.key())); + this(props, props.getString(KeyGeneratorOptions.RECORDKEY_FIELD_NAME.key()), + props.getString(KeyGeneratorOptions.PARTITIONPATH_FIELD_NAME.key())); } SimpleAvroKeyGenerator(TypedProperties props, String partitionPathField) { diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/keygen/TimestampBasedAvroKeyGenerator.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/keygen/TimestampBasedAvroKeyGenerator.java index 678591ec4..430cde81d 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/keygen/TimestampBasedAvroKeyGenerator.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/keygen/TimestampBasedAvroKeyGenerator.java @@ -88,8 +88,8 @@ public class TimestampBasedAvroKeyGenerator extends SimpleAvroKeyGenerator { } public TimestampBasedAvroKeyGenerator(TypedProperties config) throws IOException { - this(config, config.getString(KeyGeneratorOptions.RECORDKEY_FIELD.key()), - config.getString(KeyGeneratorOptions.PARTITIONPATH_FIELD.key())); + this(config, config.getString(KeyGeneratorOptions.RECORDKEY_FIELD_NAME.key()), + config.getString(KeyGeneratorOptions.PARTITIONPATH_FIELD_NAME.key())); } TimestampBasedAvroKeyGenerator(TypedProperties config, String partitionPathField) throws IOException { diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/BaseOneToTwoUpgradeHandler.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/BaseOneToTwoUpgradeHandler.java index 6a43f61c5..e3a14e487 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/BaseOneToTwoUpgradeHandler.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/BaseOneToTwoUpgradeHandler.java @@ -33,7 +33,7 @@ public abstract class BaseOneToTwoUpgradeHandler implements UpgradeHandler { public Map upgrade(HoodieWriteConfig config, HoodieEngineContext context, String instantTime) { Map tablePropsToAdd = new HashMap<>(); tablePropsToAdd.put(HoodieTableConfig.PARTITION_FIELDS, getPartitionColumns(config)); - tablePropsToAdd.put(HoodieTableConfig.RECORDKEY_FIELDS, config.getString(KeyGeneratorOptions.RECORDKEY_FIELD.key())); + tablePropsToAdd.put(HoodieTableConfig.RECORDKEY_FIELDS, config.getString(KeyGeneratorOptions.RECORDKEY_FIELD_NAME.key())); tablePropsToAdd.put(HoodieTableConfig.BASE_FILE_FORMAT, config.getString(HoodieTableConfig.BASE_FILE_FORMAT)); return tablePropsToAdd; } diff --git a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/keygen/factory/TestCreateAvroKeyGeneratorByTypeWithFactory.java b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/keygen/factory/TestCreateAvroKeyGeneratorByTypeWithFactory.java index 7983c66a6..b69d84442 100644 --- a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/keygen/factory/TestCreateAvroKeyGeneratorByTypeWithFactory.java +++ b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/keygen/factory/TestCreateAvroKeyGeneratorByTypeWithFactory.java @@ -52,9 +52,9 @@ public class TestCreateAvroKeyGeneratorByTypeWithFactory { @BeforeEach public void init() { props = new TypedProperties(); - props.put(KeyGeneratorOptions.RECORDKEY_FIELD.key(), "_row_key"); - props.put(KeyGeneratorOptions.HIVE_STYLE_PARTITIONING.key(), "true"); - props.put(KeyGeneratorOptions.PARTITIONPATH_FIELD.key(), "timestamp"); + props.put(KeyGeneratorOptions.RECORDKEY_FIELD_NAME.key(), "_row_key"); + props.put(KeyGeneratorOptions.HIVE_STYLE_PARTITIONING_ENABLE.key(), "true"); + props.put(KeyGeneratorOptions.PARTITIONPATH_FIELD_NAME.key(), "timestamp"); // for timestamp based key generator props.put("hoodie.deltastreamer.keygen.timebased.timestamp.type", "DATE_STRING"); diff --git a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/keygen/factory/TestHoodieAvroKeyGeneratorFactory.java b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/keygen/factory/TestHoodieAvroKeyGeneratorFactory.java index c17bc1bde..c3be62845 100644 --- a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/keygen/factory/TestHoodieAvroKeyGeneratorFactory.java +++ b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/keygen/factory/TestHoodieAvroKeyGeneratorFactory.java @@ -67,9 +67,9 @@ public class TestHoodieAvroKeyGeneratorFactory { private TypedProperties getCommonProps() { TypedProperties properties = new TypedProperties(); - properties.put(KeyGeneratorOptions.RECORDKEY_FIELD.key(), "_row_key"); - properties.put(KeyGeneratorOptions.HIVE_STYLE_PARTITIONING.key(), "true"); - properties.put(KeyGeneratorOptions.PARTITIONPATH_FIELD.key(), "timestamp"); + properties.put(KeyGeneratorOptions.RECORDKEY_FIELD_NAME.key(), "_row_key"); + properties.put(KeyGeneratorOptions.HIVE_STYLE_PARTITIONING_ENABLE.key(), "true"); + properties.put(KeyGeneratorOptions.PARTITIONPATH_FIELD_NAME.key(), "timestamp"); return properties; } } diff --git a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/table/upgrade/OneToTwoUpgradeHandler.java b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/table/upgrade/OneToTwoUpgradeHandler.java index 3aa9f9b27..b84ce6dd5 100644 --- a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/table/upgrade/OneToTwoUpgradeHandler.java +++ b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/table/upgrade/OneToTwoUpgradeHandler.java @@ -25,6 +25,6 @@ public class OneToTwoUpgradeHandler extends BaseOneToTwoUpgradeHandler { @Override String getPartitionColumns(HoodieWriteConfig config) { - return config.getProps().getProperty(KeyGeneratorOptions.PARTITIONPATH_FIELD.key()); + return config.getProps().getProperty(KeyGeneratorOptions.PARTITIONPATH_FIELD_NAME.key()); } } diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/keygen/ComplexKeyGenerator.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/keygen/ComplexKeyGenerator.java index 6794fc16b..8d02ce60f 100644 --- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/keygen/ComplexKeyGenerator.java +++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/keygen/ComplexKeyGenerator.java @@ -37,9 +37,9 @@ public class ComplexKeyGenerator extends BuiltinKeyGenerator { public ComplexKeyGenerator(TypedProperties props) { super(props); - this.recordKeyFields = Arrays.stream(props.getString(KeyGeneratorOptions.RECORDKEY_FIELD.key()) + this.recordKeyFields = Arrays.stream(props.getString(KeyGeneratorOptions.RECORDKEY_FIELD_NAME.key()) .split(",")).map(String::trim).filter(s -> !s.isEmpty()).collect(Collectors.toList()); - this.partitionPathFields = Arrays.stream(props.getString(KeyGeneratorOptions.PARTITIONPATH_FIELD.key()) + this.partitionPathFields = Arrays.stream(props.getString(KeyGeneratorOptions.PARTITIONPATH_FIELD_NAME.key()) .split(",")).map(String::trim).filter(s -> !s.isEmpty()).collect(Collectors.toList()); complexAvroKeyGenerator = new ComplexAvroKeyGenerator(props); } diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/keygen/CustomKeyGenerator.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/keygen/CustomKeyGenerator.java index 2ad0ace88..c43892af4 100644 --- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/keygen/CustomKeyGenerator.java +++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/keygen/CustomKeyGenerator.java @@ -53,8 +53,8 @@ public class CustomKeyGenerator extends BuiltinKeyGenerator { public CustomKeyGenerator(TypedProperties props) { super(props); - this.recordKeyFields = Arrays.stream(props.getString(KeyGeneratorOptions.RECORDKEY_FIELD.key()).split(",")).map(String::trim).collect(Collectors.toList()); - this.partitionPathFields = Arrays.stream(props.getString(KeyGeneratorOptions.PARTITIONPATH_FIELD.key()).split(",")).map(String::trim).collect(Collectors.toList()); + this.recordKeyFields = Arrays.stream(props.getString(KeyGeneratorOptions.RECORDKEY_FIELD_NAME.key()).split(",")).map(String::trim).collect(Collectors.toList()); + this.partitionPathFields = Arrays.stream(props.getString(KeyGeneratorOptions.PARTITIONPATH_FIELD_NAME.key()).split(",")).map(String::trim).collect(Collectors.toList()); customAvroKeyGenerator = new CustomAvroKeyGenerator(props); } diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/keygen/GlobalDeleteKeyGenerator.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/keygen/GlobalDeleteKeyGenerator.java index e90a1006e..391ea2c87 100644 --- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/keygen/GlobalDeleteKeyGenerator.java +++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/keygen/GlobalDeleteKeyGenerator.java @@ -39,7 +39,7 @@ public class GlobalDeleteKeyGenerator extends BuiltinKeyGenerator { private final GlobalAvroDeleteKeyGenerator globalAvroDeleteKeyGenerator; public GlobalDeleteKeyGenerator(TypedProperties config) { super(config); - this.recordKeyFields = Arrays.asList(config.getString(KeyGeneratorOptions.RECORDKEY_FIELD.key()).split(",")); + this.recordKeyFields = Arrays.asList(config.getString(KeyGeneratorOptions.RECORDKEY_FIELD_NAME.key()).split(",")); globalAvroDeleteKeyGenerator = new GlobalAvroDeleteKeyGenerator(config); } diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/keygen/NonpartitionedKeyGenerator.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/keygen/NonpartitionedKeyGenerator.java index 6a9dbafa7..1664c86f9 100644 --- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/keygen/NonpartitionedKeyGenerator.java +++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/keygen/NonpartitionedKeyGenerator.java @@ -39,7 +39,7 @@ public class NonpartitionedKeyGenerator extends BuiltinKeyGenerator { public NonpartitionedKeyGenerator(TypedProperties props) { super(props); - this.recordKeyFields = Arrays.stream(props.getString(KeyGeneratorOptions.RECORDKEY_FIELD.key()) + this.recordKeyFields = Arrays.stream(props.getString(KeyGeneratorOptions.RECORDKEY_FIELD_NAME.key()) .split(",")).map(String::trim).collect(Collectors.toList()); this.partitionPathFields = Collections.emptyList(); nonpartitionedAvroKeyGenerator = new NonpartitionedAvroKeyGenerator(props); diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/keygen/SimpleKeyGenerator.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/keygen/SimpleKeyGenerator.java index e82bf75cc..60765b05d 100644 --- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/keygen/SimpleKeyGenerator.java +++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/keygen/SimpleKeyGenerator.java @@ -36,8 +36,8 @@ public class SimpleKeyGenerator extends BuiltinKeyGenerator { private final SimpleAvroKeyGenerator simpleAvroKeyGenerator; public SimpleKeyGenerator(TypedProperties props) { - this(props, props.getString(KeyGeneratorOptions.RECORDKEY_FIELD.key()), - props.getString(KeyGeneratorOptions.PARTITIONPATH_FIELD.key())); + this(props, props.getString(KeyGeneratorOptions.RECORDKEY_FIELD_NAME.key()), + props.getString(KeyGeneratorOptions.PARTITIONPATH_FIELD_NAME.key())); } SimpleKeyGenerator(TypedProperties props, String partitionPathField) { diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/keygen/TimestampBasedKeyGenerator.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/keygen/TimestampBasedKeyGenerator.java index b4bca9f96..03d1e9242 100644 --- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/keygen/TimestampBasedKeyGenerator.java +++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/keygen/TimestampBasedKeyGenerator.java @@ -41,8 +41,8 @@ public class TimestampBasedKeyGenerator extends SimpleKeyGenerator { private final TimestampBasedAvroKeyGenerator timestampBasedAvroKeyGenerator; public TimestampBasedKeyGenerator(TypedProperties config) throws IOException { - this(config, config.getString(KeyGeneratorOptions.RECORDKEY_FIELD.key()), - config.getString(KeyGeneratorOptions.PARTITIONPATH_FIELD.key())); + this(config, config.getString(KeyGeneratorOptions.RECORDKEY_FIELD_NAME.key()), + config.getString(KeyGeneratorOptions.PARTITIONPATH_FIELD_NAME.key())); } TimestampBasedKeyGenerator(TypedProperties config, String partitionPathField) throws IOException { diff --git a/hudi-client/hudi-spark-client/src/main/scala/org/apache/hudi/HoodieSparkUtils.scala b/hudi-client/hudi-spark-client/src/main/scala/org/apache/hudi/HoodieSparkUtils.scala index 239e101d0..ed0ab9742 100644 --- a/hudi-client/hudi-spark-client/src/main/scala/org/apache/hudi/HoodieSparkUtils.scala +++ b/hudi-client/hudi-spark-client/src/main/scala/org/apache/hudi/HoodieSparkUtils.scala @@ -258,7 +258,7 @@ object HoodieSparkUtils extends SparkAdapterSupport { .mkString(",") case b: BaseKeyGenerator => b.getPartitionPathFields.asScala.mkString(",") - case _ => typedProperties.getString(KeyGeneratorOptions.PARTITIONPATH_FIELD.key()) + case _ => typedProperties.getString(KeyGeneratorOptions.PARTITIONPATH_FIELD_NAME.key()) } } diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/keygen/TestComplexKeyGenerator.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/keygen/TestComplexKeyGenerator.java index f4f6a6cef..87deef2a5 100644 --- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/keygen/TestComplexKeyGenerator.java +++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/keygen/TestComplexKeyGenerator.java @@ -39,11 +39,11 @@ public class TestComplexKeyGenerator extends KeyGeneratorTestUtilities { private TypedProperties getCommonProps(boolean getComplexRecordKey) { TypedProperties properties = new TypedProperties(); if (getComplexRecordKey) { - properties.put(KeyGeneratorOptions.RECORDKEY_FIELD.key(), "_row_key, pii_col"); + properties.put(KeyGeneratorOptions.RECORDKEY_FIELD_NAME.key(), "_row_key, pii_col"); } else { - properties.put(KeyGeneratorOptions.RECORDKEY_FIELD.key(), "_row_key"); + properties.put(KeyGeneratorOptions.RECORDKEY_FIELD_NAME.key(), "_row_key"); } - properties.put(KeyGeneratorOptions.HIVE_STYLE_PARTITIONING.key(), "true"); + properties.put(KeyGeneratorOptions.HIVE_STYLE_PARTITIONING_ENABLE.key(), "true"); return properties; } @@ -53,20 +53,20 @@ public class TestComplexKeyGenerator extends KeyGeneratorTestUtilities { private TypedProperties getPropertiesWithoutRecordKeyProp() { TypedProperties properties = new TypedProperties(); - properties.put(KeyGeneratorOptions.PARTITIONPATH_FIELD.key(), "timestamp"); + properties.put(KeyGeneratorOptions.PARTITIONPATH_FIELD_NAME.key(), "timestamp"); return properties; } private TypedProperties getWrongRecordKeyFieldProps() { TypedProperties properties = new TypedProperties(); - properties.put(KeyGeneratorOptions.PARTITIONPATH_FIELD.key(), "timestamp"); - properties.put(KeyGeneratorOptions.RECORDKEY_FIELD.key(), "_wrong_key"); + properties.put(KeyGeneratorOptions.PARTITIONPATH_FIELD_NAME.key(), "timestamp"); + properties.put(KeyGeneratorOptions.RECORDKEY_FIELD_NAME.key(), "_wrong_key"); return properties; } private TypedProperties getProps() { TypedProperties properties = getCommonProps(true); - properties.put(KeyGeneratorOptions.PARTITIONPATH_FIELD.key(), "timestamp,ts_ms"); + properties.put(KeyGeneratorOptions.PARTITIONPATH_FIELD_NAME.key(), "timestamp,ts_ms"); return properties; } @@ -105,8 +105,8 @@ public class TestComplexKeyGenerator extends KeyGeneratorTestUtilities { @Test public void testSingleValueKeyGenerator() { TypedProperties properties = new TypedProperties(); - properties.setProperty(KeyGeneratorOptions.RECORDKEY_FIELD.key(), "_row_key"); - properties.setProperty(KeyGeneratorOptions.PARTITIONPATH_FIELD.key(), "timestamp"); + properties.setProperty(KeyGeneratorOptions.RECORDKEY_FIELD_NAME.key(), "_row_key"); + properties.setProperty(KeyGeneratorOptions.PARTITIONPATH_FIELD_NAME.key(), "timestamp"); ComplexKeyGenerator compositeKeyGenerator = new ComplexKeyGenerator(properties); assertEquals(compositeKeyGenerator.getRecordKeyFields().size(), 1); assertEquals(compositeKeyGenerator.getPartitionPathFields().size(), 1); @@ -128,8 +128,8 @@ public class TestComplexKeyGenerator extends KeyGeneratorTestUtilities { @Test public void testMultipleValueKeyGenerator() { TypedProperties properties = new TypedProperties(); - properties.setProperty(KeyGeneratorOptions.RECORDKEY_FIELD.key(), "_row_key,timestamp"); - properties.setProperty(KeyGeneratorOptions.PARTITIONPATH_FIELD.key(), "rider,driver"); + properties.setProperty(KeyGeneratorOptions.RECORDKEY_FIELD_NAME.key(), "_row_key,timestamp"); + properties.setProperty(KeyGeneratorOptions.PARTITIONPATH_FIELD_NAME.key(), "rider,driver"); ComplexKeyGenerator compositeKeyGenerator = new ComplexKeyGenerator(properties); assertEquals(compositeKeyGenerator.getRecordKeyFields().size(), 2); assertEquals(compositeKeyGenerator.getPartitionPathFields().size(), 2); @@ -154,8 +154,8 @@ public class TestComplexKeyGenerator extends KeyGeneratorTestUtilities { @Test public void testMultipleValueKeyGeneratorNonPartitioned() { TypedProperties properties = new TypedProperties(); - properties.setProperty(KeyGeneratorOptions.RECORDKEY_FIELD.key(), "_row_key,timestamp"); - properties.setProperty(KeyGeneratorOptions.PARTITIONPATH_FIELD.key(), ""); + properties.setProperty(KeyGeneratorOptions.RECORDKEY_FIELD_NAME.key(), "_row_key,timestamp"); + properties.setProperty(KeyGeneratorOptions.PARTITIONPATH_FIELD_NAME.key(), ""); ComplexKeyGenerator compositeKeyGenerator = new ComplexKeyGenerator(properties); assertEquals(compositeKeyGenerator.getRecordKeyFields().size(), 2); assertEquals(compositeKeyGenerator.getPartitionPathFields().size(), 0); diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/keygen/TestCustomKeyGenerator.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/keygen/TestCustomKeyGenerator.java index 9dbb3a72c..4bfc71fa3 100644 --- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/keygen/TestCustomKeyGenerator.java +++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/keygen/TestCustomKeyGenerator.java @@ -48,53 +48,53 @@ public class TestCustomKeyGenerator extends KeyGeneratorTestUtilities { private TypedProperties getCommonProps(boolean getComplexRecordKey, boolean useKeyGeneratorClassName) { TypedProperties properties = new TypedProperties(); if (getComplexRecordKey) { - properties.put(KeyGeneratorOptions.RECORDKEY_FIELD.key(), "_row_key, pii_col"); + properties.put(KeyGeneratorOptions.RECORDKEY_FIELD_NAME.key(), "_row_key, pii_col"); } else { - properties.put(KeyGeneratorOptions.RECORDKEY_FIELD.key(), "_row_key"); + properties.put(KeyGeneratorOptions.RECORDKEY_FIELD_NAME.key(), "_row_key"); } if (useKeyGeneratorClassName) { properties.put(HoodieWriteConfig.KEYGENERATOR_CLASS_NAME.key(), CustomKeyGenerator.class.getName()); } else { properties.put(HoodieWriteConfig.KEYGENERATOR_TYPE.key(), KeyGeneratorType.CUSTOM.name()); } - properties.put(KeyGeneratorOptions.HIVE_STYLE_PARTITIONING.key(), "true"); + properties.put(KeyGeneratorOptions.HIVE_STYLE_PARTITIONING_ENABLE.key(), "true"); return properties; } private TypedProperties getPropertiesForSimpleKeyGen(boolean useKeyGeneratorClassName) { TypedProperties properties = getCommonProps(false, useKeyGeneratorClassName); - properties.put(KeyGeneratorOptions.PARTITIONPATH_FIELD.key(), "timestamp:simple"); + properties.put(KeyGeneratorOptions.PARTITIONPATH_FIELD_NAME.key(), "timestamp:simple"); return properties; } private TypedProperties getImproperPartitionFieldFormatProp(boolean useKeyGeneratorClassName) { TypedProperties properties = getCommonProps(false, useKeyGeneratorClassName); - properties.put(KeyGeneratorOptions.PARTITIONPATH_FIELD.key(), "timestamp"); + properties.put(KeyGeneratorOptions.PARTITIONPATH_FIELD_NAME.key(), "timestamp"); return properties; } private TypedProperties getInvalidPartitionKeyTypeProps(boolean useKeyGeneratorClassName) { TypedProperties properties = getCommonProps(false, useKeyGeneratorClassName); - properties.put(KeyGeneratorOptions.PARTITIONPATH_FIELD.key(), "timestamp:dummy"); + properties.put(KeyGeneratorOptions.PARTITIONPATH_FIELD_NAME.key(), "timestamp:dummy"); return properties; } private TypedProperties getComplexRecordKeyWithSimplePartitionProps(boolean useKeyGeneratorClassName) { TypedProperties properties = getCommonProps(true, useKeyGeneratorClassName); - properties.put(KeyGeneratorOptions.PARTITIONPATH_FIELD.key(), "timestamp:simple"); + properties.put(KeyGeneratorOptions.PARTITIONPATH_FIELD_NAME.key(), "timestamp:simple"); return properties; } private TypedProperties getComplexRecordKeyAndPartitionPathProps(boolean useKeyGeneratorClassName) { TypedProperties properties = getCommonProps(true, useKeyGeneratorClassName); - properties.put(KeyGeneratorOptions.PARTITIONPATH_FIELD.key(), "timestamp:simple,ts_ms:timestamp"); + properties.put(KeyGeneratorOptions.PARTITIONPATH_FIELD_NAME.key(), "timestamp:simple,ts_ms:timestamp"); populateNecessaryPropsForTimestampBasedKeyGen(properties); return properties; } private TypedProperties getPropsWithoutRecordKeyFieldProps(boolean useKeyGeneratorClassName) { TypedProperties properties = new TypedProperties(); - properties.put(KeyGeneratorOptions.PARTITIONPATH_FIELD.key(), "timestamp:simple"); + properties.put(KeyGeneratorOptions.PARTITIONPATH_FIELD_NAME.key(), "timestamp:simple"); if (useKeyGeneratorClassName) { properties.put(HoodieWriteConfig.KEYGENERATOR_CLASS_NAME.key(), CustomKeyGenerator.class.getName()); } else { @@ -111,14 +111,14 @@ public class TestCustomKeyGenerator extends KeyGeneratorTestUtilities { private TypedProperties getPropertiesForTimestampBasedKeyGen(boolean useKeyGeneratorClassName) { TypedProperties properties = getCommonProps(false, useKeyGeneratorClassName); - properties.put(KeyGeneratorOptions.PARTITIONPATH_FIELD.key(), "ts_ms:timestamp"); + properties.put(KeyGeneratorOptions.PARTITIONPATH_FIELD_NAME.key(), "ts_ms:timestamp"); populateNecessaryPropsForTimestampBasedKeyGen(properties); return properties; } private TypedProperties getPropertiesForNonPartitionedKeyGen(boolean useKeyGeneratorClassName) { TypedProperties properties = getCommonProps(false, useKeyGeneratorClassName); - properties.put(KeyGeneratorOptions.PARTITIONPATH_FIELD.key(), ""); + properties.put(KeyGeneratorOptions.PARTITIONPATH_FIELD_NAME.key(), ""); return properties; } diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/keygen/TestGlobalDeleteKeyGenerator.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/keygen/TestGlobalDeleteKeyGenerator.java index 31457bc3c..aa9568b7a 100644 --- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/keygen/TestGlobalDeleteKeyGenerator.java +++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/keygen/TestGlobalDeleteKeyGenerator.java @@ -34,29 +34,29 @@ public class TestGlobalDeleteKeyGenerator extends KeyGeneratorTestUtilities { private TypedProperties getCommonProps(boolean getComplexRecordKey) { TypedProperties properties = new TypedProperties(); if (getComplexRecordKey) { - properties.put(KeyGeneratorOptions.RECORDKEY_FIELD.key(), "_row_key,pii_col"); + properties.put(KeyGeneratorOptions.RECORDKEY_FIELD_NAME.key(), "_row_key,pii_col"); } else { - properties.put(KeyGeneratorOptions.RECORDKEY_FIELD.key(), "_row_key"); + properties.put(KeyGeneratorOptions.RECORDKEY_FIELD_NAME.key(), "_row_key"); } - properties.put(KeyGeneratorOptions.HIVE_STYLE_PARTITIONING.key(), "true"); + properties.put(KeyGeneratorOptions.HIVE_STYLE_PARTITIONING_ENABLE.key(), "true"); return properties; } private TypedProperties getPropertiesWithoutRecordKeyProp() { TypedProperties properties = new TypedProperties(); - properties.put(KeyGeneratorOptions.PARTITIONPATH_FIELD.key(), "timestamp"); + properties.put(KeyGeneratorOptions.PARTITIONPATH_FIELD_NAME.key(), "timestamp"); return properties; } private TypedProperties getWrongRecordKeyFieldProps() { TypedProperties properties = new TypedProperties(); - properties.put(KeyGeneratorOptions.RECORDKEY_FIELD.key(), "_wrong_key"); + properties.put(KeyGeneratorOptions.RECORDKEY_FIELD_NAME.key(), "_wrong_key"); return properties; } private TypedProperties getProps() { TypedProperties properties = getCommonProps(true); - properties.put(KeyGeneratorOptions.PARTITIONPATH_FIELD.key(), "timestamp,ts_ms"); + properties.put(KeyGeneratorOptions.PARTITIONPATH_FIELD_NAME.key(), "timestamp,ts_ms"); return properties; } diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/keygen/TestNonpartitionedKeyGenerator.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/keygen/TestNonpartitionedKeyGenerator.java index 99dfbdb62..0760de112 100644 --- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/keygen/TestNonpartitionedKeyGenerator.java +++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/keygen/TestNonpartitionedKeyGenerator.java @@ -37,11 +37,11 @@ public class TestNonpartitionedKeyGenerator extends KeyGeneratorTestUtilities { private TypedProperties getCommonProps(boolean getComplexRecordKey) { TypedProperties properties = new TypedProperties(); if (getComplexRecordKey) { - properties.put(KeyGeneratorOptions.RECORDKEY_FIELD.key(), "_row_key, pii_col"); + properties.put(KeyGeneratorOptions.RECORDKEY_FIELD_NAME.key(), "_row_key, pii_col"); } else { - properties.put(KeyGeneratorOptions.RECORDKEY_FIELD.key(), "_row_key"); + properties.put(KeyGeneratorOptions.RECORDKEY_FIELD_NAME.key(), "_row_key"); } - properties.put(KeyGeneratorOptions.HIVE_STYLE_PARTITIONING.key(), "true"); + properties.put(KeyGeneratorOptions.HIVE_STYLE_PARTITIONING_ENABLE.key(), "true"); return properties; } @@ -51,19 +51,19 @@ public class TestNonpartitionedKeyGenerator extends KeyGeneratorTestUtilities { private TypedProperties getPropertiesWithPartitionPathProp() { TypedProperties properties = getCommonProps(true); - properties.put(KeyGeneratorOptions.PARTITIONPATH_FIELD.key(), "timestamp,ts_ms"); + properties.put(KeyGeneratorOptions.PARTITIONPATH_FIELD_NAME.key(), "timestamp,ts_ms"); return properties; } private TypedProperties getPropertiesWithoutRecordKeyProp() { TypedProperties properties = new TypedProperties(); - properties.put(KeyGeneratorOptions.PARTITIONPATH_FIELD.key(), "timestamp"); + properties.put(KeyGeneratorOptions.PARTITIONPATH_FIELD_NAME.key(), "timestamp"); return properties; } private TypedProperties getWrongRecordKeyFieldProps() { TypedProperties properties = new TypedProperties(); - properties.put(KeyGeneratorOptions.RECORDKEY_FIELD.key(), "_wrong_key"); + properties.put(KeyGeneratorOptions.RECORDKEY_FIELD_NAME.key(), "_wrong_key"); return properties; } @@ -78,7 +78,7 @@ public class TestNonpartitionedKeyGenerator extends KeyGeneratorTestUtilities { NonpartitionedKeyGenerator keyGenerator = new NonpartitionedKeyGenerator(properties); GenericRecord record = getRecord(); Row row = KeyGeneratorTestUtilities.getRow(record); - Assertions.assertEquals(properties.getString(KeyGeneratorOptions.PARTITIONPATH_FIELD.key()), "timestamp,ts_ms"); + Assertions.assertEquals(properties.getString(KeyGeneratorOptions.PARTITIONPATH_FIELD_NAME.key()), "timestamp,ts_ms"); Assertions.assertEquals(keyGenerator.getPartitionPath(row), ""); } @@ -101,8 +101,8 @@ public class TestNonpartitionedKeyGenerator extends KeyGeneratorTestUtilities { @Test public void testSingleValueKeyGeneratorNonPartitioned() { TypedProperties properties = new TypedProperties(); - properties.setProperty(KeyGeneratorOptions.RECORDKEY_FIELD.key(), "timestamp"); - properties.setProperty(KeyGeneratorOptions.PARTITIONPATH_FIELD.key(), ""); + properties.setProperty(KeyGeneratorOptions.RECORDKEY_FIELD_NAME.key(), "timestamp"); + properties.setProperty(KeyGeneratorOptions.PARTITIONPATH_FIELD_NAME.key(), ""); NonpartitionedKeyGenerator keyGenerator = new NonpartitionedKeyGenerator(properties); assertEquals(keyGenerator.getRecordKeyFields().size(), 1); assertEquals(keyGenerator.getPartitionPathFields().size(), 0); @@ -118,8 +118,8 @@ public class TestNonpartitionedKeyGenerator extends KeyGeneratorTestUtilities { @Test public void testMultipleValueKeyGeneratorNonPartitioned1() { TypedProperties properties = new TypedProperties(); - properties.setProperty(KeyGeneratorOptions.RECORDKEY_FIELD.key(), "timestamp,driver"); - properties.setProperty(KeyGeneratorOptions.PARTITIONPATH_FIELD.key(), ""); + properties.setProperty(KeyGeneratorOptions.RECORDKEY_FIELD_NAME.key(), "timestamp,driver"); + properties.setProperty(KeyGeneratorOptions.PARTITIONPATH_FIELD_NAME.key(), ""); NonpartitionedKeyGenerator keyGenerator = new NonpartitionedKeyGenerator(properties); assertEquals(keyGenerator.getRecordKeyFields().size(), 2); assertEquals(keyGenerator.getPartitionPathFields().size(), 0); diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/keygen/TestSimpleKeyGenerator.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/keygen/TestSimpleKeyGenerator.java index 5fbbd8dac..75dca2e8d 100644 --- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/keygen/TestSimpleKeyGenerator.java +++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/keygen/TestSimpleKeyGenerator.java @@ -39,8 +39,8 @@ import static org.apache.hudi.keygen.KeyGenUtils.DEFAULT_PARTITION_PATH; public class TestSimpleKeyGenerator extends KeyGeneratorTestUtilities { private TypedProperties getCommonProps() { TypedProperties properties = new TypedProperties(); - properties.put(KeyGeneratorOptions.RECORDKEY_FIELD.key(), "_row_key"); - properties.put(KeyGeneratorOptions.HIVE_STYLE_PARTITIONING.key(), "true"); + properties.put(KeyGeneratorOptions.RECORDKEY_FIELD_NAME.key(), "_row_key"); + properties.put(KeyGeneratorOptions.HIVE_STYLE_PARTITIONING_ENABLE.key(), "true"); return properties; } @@ -50,40 +50,40 @@ public class TestSimpleKeyGenerator extends KeyGeneratorTestUtilities { private TypedProperties getPropertiesWithoutRecordKeyProp() { TypedProperties properties = new TypedProperties(); - properties.put(KeyGeneratorOptions.PARTITIONPATH_FIELD.key(), "timestamp"); + properties.put(KeyGeneratorOptions.PARTITIONPATH_FIELD_NAME.key(), "timestamp"); return properties; } private TypedProperties getWrongRecordKeyFieldProps() { TypedProperties properties = new TypedProperties(); - properties.put(KeyGeneratorOptions.PARTITIONPATH_FIELD.key(), "timestamp"); - properties.put(KeyGeneratorOptions.RECORDKEY_FIELD.key(), "_wrong_key"); + properties.put(KeyGeneratorOptions.PARTITIONPATH_FIELD_NAME.key(), "timestamp"); + properties.put(KeyGeneratorOptions.RECORDKEY_FIELD_NAME.key(), "_wrong_key"); return properties; } private TypedProperties getWrongPartitionPathFieldProps() { TypedProperties properties = new TypedProperties(); - properties.put(KeyGeneratorOptions.PARTITIONPATH_FIELD.key(), "_wrong_partition_path"); - properties.put(KeyGeneratorOptions.RECORDKEY_FIELD.key(), "_row_key"); + properties.put(KeyGeneratorOptions.PARTITIONPATH_FIELD_NAME.key(), "_wrong_partition_path"); + properties.put(KeyGeneratorOptions.RECORDKEY_FIELD_NAME.key(), "_row_key"); return properties; } private TypedProperties getComplexRecordKeyProp() { TypedProperties properties = new TypedProperties(); - properties.put(KeyGeneratorOptions.PARTITIONPATH_FIELD.key(), "timestamp"); - properties.put(KeyGeneratorOptions.RECORDKEY_FIELD.key(), "_row_key,pii_col"); + properties.put(KeyGeneratorOptions.PARTITIONPATH_FIELD_NAME.key(), "timestamp"); + properties.put(KeyGeneratorOptions.RECORDKEY_FIELD_NAME.key(), "_row_key,pii_col"); return properties; } private TypedProperties getProps() { TypedProperties properties = getCommonProps(); - properties.put(KeyGeneratorOptions.PARTITIONPATH_FIELD.key(), "timestamp"); + properties.put(KeyGeneratorOptions.PARTITIONPATH_FIELD_NAME.key(), "timestamp"); return properties; } private TypedProperties getPropsWithNestedPartitionPathField() { TypedProperties properties = getCommonProps(); - properties.put(KeyGeneratorOptions.PARTITIONPATH_FIELD.key(), "nested_col.prop1"); + properties.put(KeyGeneratorOptions.PARTITIONPATH_FIELD_NAME.key(), "nested_col.prop1"); return properties; } diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/keygen/TestTimestampBasedKeyGenerator.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/keygen/TestTimestampBasedKeyGenerator.java index 2a3841829..88a54c8ca 100644 --- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/keygen/TestTimestampBasedKeyGenerator.java +++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/keygen/TestTimestampBasedKeyGenerator.java @@ -64,9 +64,9 @@ public class TestTimestampBasedKeyGenerator { baseRow = genericRecordToRow(baseRecord); internalRow = KeyGeneratorTestUtilities.getInternalRow(baseRow); - properties.setProperty(KeyGeneratorOptions.RECORDKEY_FIELD.key(), "field1"); - properties.setProperty(KeyGeneratorOptions.PARTITIONPATH_FIELD.key(), "createTime"); - properties.setProperty(KeyGeneratorOptions.HIVE_STYLE_PARTITIONING.key(), "false"); + properties.setProperty(KeyGeneratorOptions.RECORDKEY_FIELD_NAME.key(), "field1"); + properties.setProperty(KeyGeneratorOptions.PARTITIONPATH_FIELD_NAME.key(), "createTime"); + properties.setProperty(KeyGeneratorOptions.HIVE_STYLE_PARTITIONING_ENABLE.key(), "false"); } private TypedProperties getBaseKeyConfig(String timestampType, String dateFormat, String timezone, String scalarType) { diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/keygen/factory/TestCreateKeyGeneratorByTypeWithFactory.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/keygen/factory/TestCreateKeyGeneratorByTypeWithFactory.java index 10bc5066d..ad3edd449 100644 --- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/keygen/factory/TestCreateKeyGeneratorByTypeWithFactory.java +++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/keygen/factory/TestCreateKeyGeneratorByTypeWithFactory.java @@ -53,9 +53,9 @@ public class TestCreateKeyGeneratorByTypeWithFactory { @BeforeEach public void init() { props = new TypedProperties(); - props.put(KeyGeneratorOptions.RECORDKEY_FIELD.key(), "_row_key"); - props.put(KeyGeneratorOptions.HIVE_STYLE_PARTITIONING.key(), "true"); - props.put(KeyGeneratorOptions.PARTITIONPATH_FIELD.key(), "timestamp"); + props.put(KeyGeneratorOptions.RECORDKEY_FIELD_NAME.key(), "_row_key"); + props.put(KeyGeneratorOptions.HIVE_STYLE_PARTITIONING_ENABLE.key(), "true"); + props.put(KeyGeneratorOptions.PARTITIONPATH_FIELD_NAME.key(), "timestamp"); // for timestamp based key generator props.put("hoodie.deltastreamer.keygen.timebased.timestamp.type", "DATE_STRING"); diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/keygen/factory/TestHoodieSparkKeyGeneratorFactory.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/keygen/factory/TestHoodieSparkKeyGeneratorFactory.java index 682329a76..dffe1eaa9 100644 --- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/keygen/factory/TestHoodieSparkKeyGeneratorFactory.java +++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/keygen/factory/TestHoodieSparkKeyGeneratorFactory.java @@ -74,9 +74,9 @@ public class TestHoodieSparkKeyGeneratorFactory { private TypedProperties getCommonProps() { TypedProperties properties = new TypedProperties(); - properties.put(KeyGeneratorOptions.RECORDKEY_FIELD.key(), "_row_key"); - properties.put(KeyGeneratorOptions.HIVE_STYLE_PARTITIONING.key(), "true"); - properties.put(KeyGeneratorOptions.PARTITIONPATH_FIELD.key(), "timestamp"); + properties.put(KeyGeneratorOptions.RECORDKEY_FIELD_NAME.key(), "_row_key"); + properties.put(KeyGeneratorOptions.HIVE_STYLE_PARTITIONING_ENABLE.key(), "true"); + properties.put(KeyGeneratorOptions.PARTITIONPATH_FIELD_NAME.key(), "timestamp"); return properties; } } diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/upgrade/TestUpgradeDowngrade.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/upgrade/TestUpgradeDowngrade.java index 123a33c3d..792da4e08 100644 --- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/upgrade/TestUpgradeDowngrade.java +++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/upgrade/TestUpgradeDowngrade.java @@ -231,8 +231,8 @@ public class TestUpgradeDowngrade extends HoodieClientTestBase { } private void addNewTableParamsToProps(Map params) { - params.put(KeyGeneratorOptions.RECORDKEY_FIELD.key(), "uuid"); - params.put(KeyGeneratorOptions.PARTITIONPATH_FIELD.key(), "partition_path"); + params.put(KeyGeneratorOptions.RECORDKEY_FIELD_NAME.key(), "uuid"); + params.put(KeyGeneratorOptions.PARTITIONPATH_FIELD_NAME.key(), "partition_path"); params.put(HoodieTableConfig.NAME.key(), metaClient.getTableConfig().getTableName()); params.put(BASE_FILE_FORMAT.key(), BASE_FILE_FORMAT.defaultValue().name()); } @@ -262,8 +262,8 @@ public class TestUpgradeDowngrade extends HoodieClientTestBase { private void assertTableProps(HoodieWriteConfig cfg) { HoodieTableConfig tableConfig = metaClient.getTableConfig(); Properties originalProps = cfg.getProps(); - assertEquals(tableConfig.getPartitionFieldProp(), originalProps.getProperty(KeyGeneratorOptions.PARTITIONPATH_FIELD.key())); - assertEquals(tableConfig.getRecordKeyFieldProp(), originalProps.getProperty(KeyGeneratorOptions.RECORDKEY_FIELD.key())); + assertEquals(tableConfig.getPartitionFieldProp(), originalProps.getProperty(KeyGeneratorOptions.PARTITIONPATH_FIELD_NAME.key())); + assertEquals(tableConfig.getRecordKeyFieldProp(), originalProps.getProperty(KeyGeneratorOptions.RECORDKEY_FIELD_NAME.key())); assertEquals(tableConfig.getTableName(), cfg.getTableName()); assertEquals(tableConfig.getBaseFileFormat().name(), originalProps.getProperty(BASE_FILE_FORMAT.key())); } @@ -289,8 +289,8 @@ public class TestUpgradeDowngrade extends HoodieClientTestBase { // set table configs HoodieTableConfig tableConfig = metaClient.getTableConfig(); tableConfig.setValue(HoodieTableConfig.NAME, cfg.getTableName()); - tableConfig.setValue(HoodieTableConfig.PARTITION_FIELDS, cfg.getString(KeyGeneratorOptions.PARTITIONPATH_FIELD.key())); - tableConfig.setValue(HoodieTableConfig.RECORDKEY_FIELDS, cfg.getString(KeyGeneratorOptions.RECORDKEY_FIELD.key())); + tableConfig.setValue(HoodieTableConfig.PARTITION_FIELDS, cfg.getString(KeyGeneratorOptions.PARTITIONPATH_FIELD_NAME.key())); + tableConfig.setValue(HoodieTableConfig.RECORDKEY_FIELDS, cfg.getString(KeyGeneratorOptions.RECORDKEY_FIELD_NAME.key())); tableConfig.setValue(BASE_FILE_FORMAT, cfg.getString(BASE_FILE_FORMAT)); } diff --git a/hudi-common/src/main/java/org/apache/hudi/keygen/BaseKeyGenerator.java b/hudi-common/src/main/java/org/apache/hudi/keygen/BaseKeyGenerator.java index 654702ef5..88092dacb 100644 --- a/hudi-common/src/main/java/org/apache/hudi/keygen/BaseKeyGenerator.java +++ b/hudi-common/src/main/java/org/apache/hudi/keygen/BaseKeyGenerator.java @@ -37,8 +37,8 @@ public abstract class BaseKeyGenerator extends KeyGenerator { super(config); this.encodePartitionPath = config.getBoolean(KeyGeneratorOptions.URL_ENCODE_PARTITIONING.key(), Boolean.parseBoolean(KeyGeneratorOptions.URL_ENCODE_PARTITIONING.defaultValue())); - this.hiveStylePartitioning = config.getBoolean(KeyGeneratorOptions.HIVE_STYLE_PARTITIONING.key(), - Boolean.parseBoolean(KeyGeneratorOptions.HIVE_STYLE_PARTITIONING.defaultValue())); + this.hiveStylePartitioning = config.getBoolean(KeyGeneratorOptions.HIVE_STYLE_PARTITIONING_ENABLE.key(), + Boolean.parseBoolean(KeyGeneratorOptions.HIVE_STYLE_PARTITIONING_ENABLE.defaultValue())); } /** diff --git a/hudi-common/src/main/java/org/apache/hudi/keygen/constant/KeyGeneratorOptions.java b/hudi-common/src/main/java/org/apache/hudi/keygen/constant/KeyGeneratorOptions.java index ab51b8e2a..3e369a54b 100644 --- a/hudi-common/src/main/java/org/apache/hudi/keygen/constant/KeyGeneratorOptions.java +++ b/hudi-common/src/main/java/org/apache/hudi/keygen/constant/KeyGeneratorOptions.java @@ -36,21 +36,21 @@ public class KeyGeneratorOptions extends HoodieConfig { .defaultValue("false") .withDocumentation("Should we url encode the partition path value, before creating the folder structure."); - public static final ConfigProperty HIVE_STYLE_PARTITIONING = ConfigProperty + public static final ConfigProperty HIVE_STYLE_PARTITIONING_ENABLE = ConfigProperty .key("hoodie.datasource.write.hive_style_partitioning") .defaultValue("false") .withDocumentation("Flag to indicate whether to use Hive style partitioning.\n" + "If set true, the names of partition folders follow = format.\n" + "By default false (the names of partition folders are only partition values)"); - public static final ConfigProperty RECORDKEY_FIELD = ConfigProperty + public static final ConfigProperty RECORDKEY_FIELD_NAME = ConfigProperty .key("hoodie.datasource.write.recordkey.field") .defaultValue("uuid") .withDocumentation("Record key field. Value to be used as the `recordKey` component of `HoodieKey`.\n" + "Actual value will be obtained by invoking .toString() on the field value. Nested fields can be specified using\n" + "the dot notation eg: `a.b.c`"); - public static final ConfigProperty PARTITIONPATH_FIELD = ConfigProperty + public static final ConfigProperty PARTITIONPATH_FIELD_NAME = ConfigProperty .key("hoodie.datasource.write.partitionpath.field") .defaultValue("partitionpath") .withDocumentation("Partition path field. Value to be used at the partitionPath component of HoodieKey. " @@ -67,24 +67,24 @@ public class KeyGeneratorOptions extends HoodieConfig { @Deprecated public static final String DEFAULT_URL_ENCODE_PARTITIONING_OPT_VAL = URL_ENCODE_PARTITIONING.defaultValue(); /** - * @deprecated Use {@link #HIVE_STYLE_PARTITIONING} and its methods. + * @deprecated Use {@link #HIVE_STYLE_PARTITIONING_ENABLE} and its methods. */ @Deprecated - public static final String HIVE_STYLE_PARTITIONING_OPT_KEY = HIVE_STYLE_PARTITIONING.key(); + public static final String HIVE_STYLE_PARTITIONING_OPT_KEY = HIVE_STYLE_PARTITIONING_ENABLE.key(); /** - * @deprecated Use {@link #HIVE_STYLE_PARTITIONING} and its methods. + * @deprecated Use {@link #HIVE_STYLE_PARTITIONING_ENABLE} and its methods. */ @Deprecated - public static final String DEFAULT_HIVE_STYLE_PARTITIONING_OPT_VAL = HIVE_STYLE_PARTITIONING.defaultValue(); + public static final String DEFAULT_HIVE_STYLE_PARTITIONING_OPT_VAL = HIVE_STYLE_PARTITIONING_ENABLE.defaultValue(); /** - * @deprecated Use {@link #RECORDKEY_FIELD} and its methods. + * @deprecated Use {@link #RECORDKEY_FIELD_NAME} and its methods. */ @Deprecated - public static final String RECORDKEY_FIELD_OPT_KEY = RECORDKEY_FIELD.key(); + public static final String RECORDKEY_FIELD_OPT_KEY = RECORDKEY_FIELD_NAME.key(); /** - * @deprecated Use {@link #PARTITIONPATH_FIELD} and its methods. + * @deprecated Use {@link #PARTITIONPATH_FIELD_NAME} and its methods. */ @Deprecated - public static final String PARTITIONPATH_FIELD_OPT_KEY = PARTITIONPATH_FIELD.key(); + public static final String PARTITIONPATH_FIELD_OPT_KEY = PARTITIONPATH_FIELD_NAME.key(); } diff --git a/hudi-flink/src/main/java/org/apache/hudi/configuration/FlinkOptions.java b/hudi-flink/src/main/java/org/apache/hudi/configuration/FlinkOptions.java index 72e8fccb9..a1c441735 100644 --- a/hudi-flink/src/main/java/org/apache/hudi/configuration/FlinkOptions.java +++ b/hudi-flink/src/main/java/org/apache/hudi/configuration/FlinkOptions.java @@ -280,7 +280,7 @@ public class FlinkOptions extends HoodieConfig { + "By default true (in favor of streaming progressing over data integrity)"); public static final ConfigOption RECORD_KEY_FIELD = ConfigOptions - .key(KeyGeneratorOptions.RECORDKEY_FIELD.key()) + .key(KeyGeneratorOptions.RECORDKEY_FIELD_NAME.key()) .stringType() .defaultValue("uuid") .withDescription("Record key field. Value to be used as the `recordKey` component of `HoodieKey`.\n" @@ -288,7 +288,7 @@ public class FlinkOptions extends HoodieConfig { + "the dot notation eg: `a.b.c`"); public static final ConfigOption PARTITION_PATH_FIELD = ConfigOptions - .key(KeyGeneratorOptions.PARTITIONPATH_FIELD.key()) + .key(KeyGeneratorOptions.PARTITIONPATH_FIELD_NAME.key()) .stringType() .defaultValue("") .withDescription("Partition path field. Value to be used at the `partitionPath` component of `HoodieKey`.\n" @@ -301,7 +301,7 @@ public class FlinkOptions extends HoodieConfig { .withDescription("Whether to encode the partition path url, default false"); public static final ConfigOption HIVE_STYLE_PARTITIONING = ConfigOptions - .key(KeyGeneratorOptions.HIVE_STYLE_PARTITIONING.key()) + .key(KeyGeneratorOptions.HIVE_STYLE_PARTITIONING_ENABLE.key()) .booleanType() .defaultValue(false) .withDescription("Whether to use Hive style partitioning.\n" diff --git a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/DataSourceOptions.scala b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/DataSourceOptions.scala index 9881e431a..3c7fe7daf 100644 --- a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/DataSourceOptions.scala +++ b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/DataSourceOptions.scala @@ -65,7 +65,7 @@ object DataSourceReadOptions { .noDefaultValue() .withDocumentation("Comma separated list of file paths to read within a Hudi table.") - val READ_PRE_COMBINE_FIELD = HoodieWriteConfig.PRECOMBINE_FIELD + val READ_PRE_COMBINE_FIELD = HoodieWriteConfig.PRECOMBINE_FIELD_NAME val ENABLE_HOODIE_FILE_INDEX: ConfigProperty[Boolean] = ConfigProperty .key("hoodie.file.index.enable") @@ -238,7 +238,7 @@ object DataSourceWriteOptions { * key value, we will pick the one with the largest value for the precombine field, * determined by Object.compareTo(..) */ - val PRECOMBINE_FIELD = HoodieWriteConfig.PRECOMBINE_FIELD + val PRECOMBINE_FIELD = HoodieWriteConfig.PRECOMBINE_FIELD_NAME /** * Payload class used. Override this, if you like to roll your own merge logic, when upserting/inserting. @@ -252,20 +252,20 @@ object DataSourceWriteOptions { * the dot notation eg: `a.b.c` * */ - val RECORDKEY_FIELD = KeyGeneratorOptions.RECORDKEY_FIELD + val RECORDKEY_FIELD = KeyGeneratorOptions.RECORDKEY_FIELD_NAME /** * Partition path field. Value to be used at the `partitionPath` component of `HoodieKey`. Actual * value obtained by invoking .toString() */ - val PARTITIONPATH_FIELD = KeyGeneratorOptions.PARTITIONPATH_FIELD + val PARTITIONPATH_FIELD = KeyGeneratorOptions.PARTITIONPATH_FIELD_NAME /** * Flag to indicate whether to use Hive style partitioning. * If set true, the names of partition folders follow = format. * By default false (the names of partition folders are only partition values) */ - val HIVE_STYLE_PARTITIONING = KeyGeneratorOptions.HIVE_STYLE_PARTITIONING + val HIVE_STYLE_PARTITIONING = KeyGeneratorOptions.HIVE_STYLE_PARTITIONING_ENABLE val KEYGENERATOR_CLASS_NAME = ConfigProperty.key("hoodie.datasource.write.keygenerator.class") .defaultValue(classOf[SimpleKeyGenerator].getName) @@ -565,7 +565,7 @@ object DataSourceWriteOptions { val DEFAULT_ENABLE_ROW_WRITER_OPT_VAL = ENABLE_ROW_WRITER.defaultValue() /** @deprecated Use {@link HIVE_STYLE_PARTITIONING} and its methods instead */ @Deprecated - val HIVE_STYLE_PARTITIONING_OPT_KEY = KeyGeneratorOptions.HIVE_STYLE_PARTITIONING.key() + val HIVE_STYLE_PARTITIONING_OPT_KEY = KeyGeneratorOptions.HIVE_STYLE_PARTITIONING_ENABLE.key() /** @deprecated Use {@link HIVE_STYLE_PARTITIONING} and its methods instead */ @Deprecated val DEFAULT_HIVE_STYLE_PARTITIONING_OPT_VAL = HIVE_STYLE_PARTITIONING.defaultValue() @@ -604,13 +604,13 @@ object DataSourceWriteOptions { /** @deprecated Use {@link RECORDKEY_FIELD} and its methods instead */ @Deprecated - val RECORDKEY_FIELD_OPT_KEY = KeyGeneratorOptions.RECORDKEY_FIELD.key() + val RECORDKEY_FIELD_OPT_KEY = KeyGeneratorOptions.RECORDKEY_FIELD_NAME.key() /** @deprecated Use {@link RECORDKEY_FIELD} and its methods instead */ @Deprecated val DEFAULT_RECORDKEY_FIELD_OPT_VAL = RECORDKEY_FIELD.defaultValue() /** @deprecated Use {@link PARTITIONPATH_FIELD} and its methods instead */ @Deprecated - val PARTITIONPATH_FIELD_OPT_KEY = KeyGeneratorOptions.PARTITIONPATH_FIELD.key() + val PARTITIONPATH_FIELD_OPT_KEY = KeyGeneratorOptions.PARTITIONPATH_FIELD_NAME.key() /** @deprecated Use {@link PARTITIONPATH_FIELD} and its methods instead */ @Deprecated val DEFAULT_PARTITIONPATH_FIELD_OPT_VAL = PARTITIONPATH_FIELD.defaultValue() @@ -620,7 +620,7 @@ object DataSourceWriteOptions { val TABLE_NAME_OPT_KEY = TABLE_NAME.key() /** @deprecated Use {@link PRECOMBINE_FIELD} and its methods instead */ @Deprecated - val PRECOMBINE_FIELD_OPT_KEY = HoodieWriteConfig.PRECOMBINE_FIELD.key() + val PRECOMBINE_FIELD_OPT_KEY = HoodieWriteConfig.PRECOMBINE_FIELD_NAME.key() /** @deprecated Use {@link PRECOMBINE_FIELD} and its methods instead */ @Deprecated val DEFAULT_PRECOMBINE_FIELD_OPT_VAL = PRECOMBINE_FIELD.defaultValue()