lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-1448] Hudi dla sync support skip rt table syncing (#2324)

Browse Source
This commit is contained in:
lw0090
2020-12-14 23:25:10 +08:00
committed by GitHub
parent 11bc1fe6f4
commit facde4c16f
3 changed files with 9 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
hudi-sync/hudi-dla-sync/src/main/java/org/apache/hudi/dla/DLASyncConfig.java
View File

@@ -62,6 +62,9 @@ public class DLASyncConfig implements Serializable {
@Parameter(names = {"--skip-ro-suffix"}, description = "Skip the `_ro` suffix for Read optimized table, when registering")
public Boolean skipROSuffix = false;
@Parameter(names = {"--skip-rt-sync"}, description = "Skip the RT table syncing")
public Boolean skipRTSync = false;
@Parameter(names = {"--hive-style-partitioning"}, description = "Use DLA hive style partitioning, true if like the following style: field1=value1/field2=value2")
public Boolean useDLASyncHiveStylePartitioning = false;
@@ -83,6 +86,7 @@ public class DLASyncConfig implements Serializable {
newConfig.partitionValueExtractorClass = cfg.partitionValueExtractorClass;
newConfig.assumeDatePartitioning = cfg.assumeDatePartitioning;
newConfig.skipROSuffix = cfg.skipROSuffix;
newConfig.skipRTSync = cfg.skipRTSync;
newConfig.useDLASyncHiveStylePartitioning = cfg.useDLASyncHiveStylePartitioning;
newConfig.supportTimestamp = cfg.supportTimestamp;
return newConfig;

4
hudi-sync/hudi-dla-sync/src/main/java/org/apache/hudi/dla/DLASyncTool.java
View File

@@ -93,7 +93,9 @@ public class DLASyncTool extends AbstractSyncTool {
// sync a RO table for MOR
syncHoodieTable(roTableTableName.get(), false);
// sync a RT table for MOR
syncHoodieTable(snapshotTableName, true);
if (!cfg.skipRTSync) {
syncHoodieTable(snapshotTableName, true);
}
break;
default:
LOG.error("Unknown table type " + hoodieDLAClient.getTableType());

2
hudi-sync/hudi-dla-sync/src/main/java/org/apache/hudi/dla/util/Utils.java
View File

@@ -36,6 +36,7 @@ public class Utils {
public static String DLA_PARTITION_EXTRACTOR_CLASS_OPT_KEY = "hoodie.datasource.dla_sync.partition_extractor_class";
public static String DLA_ASSUME_DATE_PARTITIONING = "hoodie.datasource.dla_sync.assume_date_partitioning";
public static String DLA_SKIP_RO_SUFFIX = "hoodie.datasource.dla_sync.skip_ro_suffix";
public static String DLA_SKIP_RT_SYNC = "hoodie.datasource.dla_sync.skip_rt_sync";
public static String DLA_SYNC_HIVE_STYLE_PARTITIONING = "hoodie.datasource.dla_sync.hive.style.partitioning";
public static Properties configToProperties(DLASyncConfig cfg) {
@@ -69,6 +70,7 @@ public class Utils {
config.partitionValueExtractorClass = properties.getProperty(DLA_PARTITION_EXTRACTOR_CLASS_OPT_KEY);
config.assumeDatePartitioning = Boolean.parseBoolean(properties.getProperty(DLA_ASSUME_DATE_PARTITIONING, "false"));
config.skipROSuffix = Boolean.parseBoolean(properties.getProperty(DLA_SKIP_RO_SUFFIX, "false"));
config.skipRTSync = Boolean.parseBoolean(properties.getProperty(DLA_SKIP_RT_SYNC, "false"));
config.useDLASyncHiveStylePartitioning = Boolean.parseBoolean(properties.getProperty(DLA_SYNC_HIVE_STYLE_PARTITIONING, "false"));
return config;
}