lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-3771] flink supports sync table information to aws glue (#5202)

Browse Source
This commit is contained in:
todd5167
2022-04-02 21:16:10 +08:00
committed by GitHub
parent 020786a5f9
commit eef3f9c74a
4 changed files with 12 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
hudi-aws/src/main/java/org/apache/hudi/aws/sync/AwsGlueCatalogSyncTool.java
View File

@@ -45,7 +45,7 @@ public class AwsGlueCatalogSyncTool extends HiveSyncTool {
super(props, new HiveConf(conf, HiveConf.class), fs); super(props, new HiveConf(conf, HiveConf.class), fs);
} }
private AwsGlueCatalogSyncTool(HiveSyncConfig hiveSyncConfig, HiveConf hiveConf, FileSystem fs) { public AwsGlueCatalogSyncTool(HiveSyncConfig hiveSyncConfig, HiveConf hiveConf, FileSystem fs) {
super(hiveSyncConfig, hiveConf, fs); super(hiveSyncConfig, hiveConf, fs);
} }

6
hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/utils/HiveSyncContext.java
View File

@@ -18,10 +18,12 @@
package org.apache.hudi.sink.utils; package org.apache.hudi.sink.utils;
import org.apache.hudi.aws.sync.AwsGlueCatalogSyncTool;
import org.apache.hudi.common.fs.FSUtils; import org.apache.hudi.common.fs.FSUtils;
import org.apache.hudi.configuration.FlinkOptions; import org.apache.hudi.configuration.FlinkOptions;
import org.apache.hudi.hive.HiveSyncConfig; import org.apache.hudi.hive.HiveSyncConfig;
import org.apache.hudi.hive.HiveSyncTool; import org.apache.hudi.hive.HiveSyncTool;
import org.apache.hudi.hive.ddl.HiveSyncMode;
import org.apache.hudi.table.format.FilePathUtils; import org.apache.hudi.table.format.FilePathUtils;
import org.apache.hudi.util.StreamerUtil; import org.apache.hudi.util.StreamerUtil;
@@ -48,6 +50,10 @@ public class HiveSyncContext {
} }
public HiveSyncTool hiveSyncTool() { public HiveSyncTool hiveSyncTool() {
HiveSyncMode syncMode = HiveSyncMode.of(syncConfig.syncMode);
if (syncMode == HiveSyncMode.GLUE) {
return new AwsGlueCatalogSyncTool(this.syncConfig, this.hiveConf, this.fs);
}
return new HiveSyncTool(this.syncConfig, this.hiveConf, this.fs); return new HiveSyncTool(this.syncConfig, this.hiveConf, this.fs);
} }

2
hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/HiveSyncConfig.java
View File

@@ -55,7 +55,7 @@ public class HiveSyncConfig extends HoodieSyncConfig {
@Parameter(names = {"--use-jdbc"}, description = "Hive jdbc connect url") @Parameter(names = {"--use-jdbc"}, description = "Hive jdbc connect url")
public Boolean useJdbc; public Boolean useJdbc;
@Parameter(names = {"--sync-mode"}, description = "Mode to choose for Hive ops. Valid values are hms, jdbc and hiveql") @Parameter(names = {"--sync-mode"}, description = "Mode to choose for Hive ops. Valid values are hms,glue,jdbc and hiveql")
public String syncMode; public String syncMode;
@Parameter(names = {"--auto-create-database"}, description = "Auto create hive database") @Parameter(names = {"--auto-create-database"}, description = "Auto create hive database")

4
hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/ddl/HiveSyncMode.java
View File

@@ -26,6 +26,10 @@ public enum HiveSyncMode {
* The HMS mode use the hive meta client to sync metadata. * The HMS mode use the hive meta client to sync metadata.
*/ */
HMS, HMS,
/**
* The GLUE mode use the glue client to sync metadata.
*/
GLUE,
/** /**
* The HIVEQL mode execute hive ql to sync metadata. * The HIVEQL mode execute hive ql to sync metadata.
*/ */