lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[MINOR] Refactor hive sync tool to reduce duplicate code (#3276)

Browse Source 
* [MINOR] Refactor hive sync tool to reduce duplicate code
This commit is contained in:
vinoyang
2021-07-15 23:54:38 +08:00
committed by GitHub
parent 23a4a96eb4
commit a62a6cff32
2 changed files with 20 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files

32
hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/HiveSyncTool.java
View File

@@ -113,20 +113,7 @@ public class HiveSyncTool extends AbstractSyncTool {
public void syncHoodieTable() { public void syncHoodieTable() {
try { try {
if (hoodieHiveClient != null) { if (hoodieHiveClient != null) {
switch (hoodieHiveClient.getTableType()) { doSync();
case COPY_ON_WRITE:
syncHoodieTable(snapshotTableName, false, false);
break;
case MERGE_ON_READ:
// sync a RO table for MOR
syncHoodieTable(roTableName.get(), false, true);
// sync a RT table for MOR
syncHoodieTable(snapshotTableName, true, false);
break;
default:
LOG.error("Unknown table type " + hoodieHiveClient.getTableType());
throw new InvalidTableException(hoodieHiveClient.getBasePath());
}
} }
} catch (RuntimeException re) { } catch (RuntimeException re) {
throw new HoodieException("Got runtime exception when hive syncing " + cfg.tableName, re); throw new HoodieException("Got runtime exception when hive syncing " + cfg.tableName, re);
@@ -137,6 +124,23 @@ public class HiveSyncTool extends AbstractSyncTool {
} }
} }
protected void doSync() {
switch (hoodieHiveClient.getTableType()) {
case COPY_ON_WRITE:
syncHoodieTable(snapshotTableName, false, false);
break;
case MERGE_ON_READ:
// sync a RO table for MOR
syncHoodieTable(roTableName.get(), false, true);
// sync a RT table for MOR
syncHoodieTable(snapshotTableName, true, false);
break;
default:
LOG.error("Unknown table type " + hoodieHiveClient.getTableType());
throw new InvalidTableException(hoodieHiveClient.getBasePath());
}
}
protected void syncHoodieTable(String tableName, boolean useRealtimeInputFormat, protected void syncHoodieTable(String tableName, boolean useRealtimeInputFormat,
boolean readAsOptimized) { boolean readAsOptimized) {
LOG.info("Trying to sync hoodie table " + tableName + " with base path " + hoodieHiveClient.getBasePath() LOG.info("Trying to sync hoodie table " + tableName + " with base path " + hoodieHiveClient.getBasePath()

16
hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/replication/GlobalHiveSyncTool.java
View File

@@ -21,7 +21,6 @@ package org.apache.hudi.hive.replication;
import org.apache.hudi.common.fs.FSUtils; import org.apache.hudi.common.fs.FSUtils;
import org.apache.hudi.common.model.HoodieTableType; import org.apache.hudi.common.model.HoodieTableType;
import org.apache.hudi.common.util.Option; import org.apache.hudi.common.util.Option;
import org.apache.hudi.exception.InvalidTableException;
import org.apache.hudi.hive.HiveSyncTool; import org.apache.hudi.hive.HiveSyncTool;
import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configuration;
@@ -43,20 +42,7 @@ public class GlobalHiveSyncTool extends HiveSyncTool {
@Override @Override
public void syncHoodieTable() { public void syncHoodieTable() {
switch (hoodieHiveClient.getTableType()) { doSync();
case COPY_ON_WRITE:
syncHoodieTable(snapshotTableName, false, false);
break;
case MERGE_ON_READ:
// sync a RO table for MOR
syncHoodieTable(roTableName.get(), false, true);
// sync a RT table for MOR
syncHoodieTable(snapshotTableName, true, false);
break;
default:
LOG.error("Unknown table type " + hoodieHiveClient.getTableType());
throw new InvalidTableException(hoodieHiveClient.getBasePath());
}
} }
@Override @Override