lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-2837] Add support for using database name in incremental query (#4083)

Browse Source
This commit is contained in:
董可伦
2022-01-23 14:11:27 +08:00
committed by GitHub
parent 64b1426005
commit 56cd8ffae0
19 changed files with 330 additions and 63 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableConfig.java
View File

@@ -76,6 +76,12 @@ public class HoodieTableConfig extends HoodieConfig {
public static final String HOODIE_PROPERTIES_FILE = "hoodie.properties";
public static final String HOODIE_PROPERTIES_FILE_BACKUP = "hoodie.properties.backup";
public static final ConfigProperty<String> DATABASE_NAME = ConfigProperty
.key("hoodie.database.name")
.noDefaultValue()
.withDocumentation("Database name that will be used for incremental query.If different databases have the same table name during incremental query, "
+ "we can set it to limit the table name under a specific database");
public static final ConfigProperty<String> NAME = ConfigProperty
.key("hoodie.table.name")
.noDefaultValue()
@@ -422,6 +428,13 @@ public class HoodieTableConfig extends HoodieConfig {
}
}
/**
* Read the database name.
*/
public String getDatabaseName() {
return getString(DATABASE_NAME);
}
/**
* Read the table name.
*/

12
hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableMetaClient.java
View File

@@ -624,6 +624,7 @@ public class HoodieTableMetaClient implements Serializable {
public static class PropertyBuilder {
private HoodieTableType tableType;
private String databaseName;
private String tableName;
private String tableCreateSchema;
private String recordKeyFields;
@@ -655,6 +656,11 @@ public class HoodieTableMetaClient implements Serializable {
return setTableType(HoodieTableType.valueOf(tableType));
}
public PropertyBuilder setDatabaseName(String databaseName) {
this.databaseName = databaseName;
return this;
}
public PropertyBuilder setTableName(String tableName) {
this.tableName = tableName;
return this;
@@ -753,6 +759,9 @@ public class HoodieTableMetaClient implements Serializable {
public PropertyBuilder fromProperties(Properties properties) {
HoodieConfig hoodieConfig = new HoodieConfig(properties);
if (hoodieConfig.contains(HoodieTableConfig.DATABASE_NAME)) {
setDatabaseName(hoodieConfig.getString(HoodieTableConfig.DATABASE_NAME));
}
if (hoodieConfig.contains(HoodieTableConfig.NAME)) {
setTableName(hoodieConfig.getString(HoodieTableConfig.NAME));
}
@@ -819,6 +828,9 @@ public class HoodieTableMetaClient implements Serializable {
ValidationUtils.checkArgument(tableName != null, "tableName is null");
HoodieTableConfig tableConfig = new HoodieTableConfig();
if (databaseName != null) {
tableConfig.setValue(HoodieTableConfig.DATABASE_NAME, databaseName);
}
tableConfig.setValue(HoodieTableConfig.NAME, tableName);
tableConfig.setValue(HoodieTableConfig.TYPE, tableType.name());
tableConfig.setValue(HoodieTableConfig.VERSION,

22
hudi-common/src/test/java/org/apache/hudi/common/testutils/HoodieTestUtils.java
View File

@@ -46,6 +46,7 @@ import java.util.UUID;
*/
public class HoodieTestUtils {
public static final String HOODIE_DATABASE = "test_incremental";
public static final String RAW_TRIPS_TEST_NAME = "raw_trips";
public static final String DEFAULT_WRITE_TOKEN = "1-0-1";
public static final int DEFAULT_LOG_VERSION = 1;
@@ -91,6 +92,14 @@ public class HoodieTestUtils {
return init(hadoopConf, basePath, tableType, properties);
}
public static HoodieTableMetaClient init(Configuration hadoopConf, String basePath, HoodieTableType tableType,
HoodieFileFormat baseFileFormat, String databaseName)
throws IOException {
Properties properties = new Properties();
properties.setProperty(HoodieTableConfig.BASE_FILE_FORMAT.key(), baseFileFormat.toString());
return init(hadoopConf, basePath, tableType, properties, databaseName);
}
public static HoodieTableMetaClient init(Configuration hadoopConf, String basePath, HoodieTableType tableType,
HoodieFileFormat baseFileFormat)
throws IOException {
@@ -111,6 +120,19 @@ public class HoodieTestUtils {
return HoodieTableMetaClient.initTableAndGetMetaClient(hadoopConf, basePath, properties);
}
public static HoodieTableMetaClient init(Configuration hadoopConf, String basePath, HoodieTableType tableType,
Properties properties, String databaseName)
throws IOException {
properties = HoodieTableMetaClient.withPropertyBuilder()
.setDatabaseName(databaseName)
.setTableName(RAW_TRIPS_TEST_NAME)
.setTableType(tableType)
.setPayloadClass(HoodieAvroPayload.class)
.fromProperties(properties)
.build();
return HoodieTableMetaClient.initTableAndGetMetaClient(hadoopConf, basePath, properties);
}
public static HoodieTableMetaClient init(String basePath, HoodieTableType tableType, String bootstrapBasePath, HoodieFileFormat baseFileFormat) throws IOException {
Properties props = new Properties();
props.setProperty(HoodieTableConfig.BOOTSTRAP_BASE_PATH.key(), bootstrapBasePath);