lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-325] Fix Hive partition error for updated HDFS Hudi table (#1001)

Browse Source
This commit is contained in:
wenningd
2019-11-26 21:18:39 -08:00
committed by vinoth chandar
parent 60fed21dc7
commit d6e83e8f49
2 changed files with 13 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
hudi-common/src/main/java/org/apache/hudi/common/util/FSUtils.java
View File

@@ -530,6 +530,13 @@ public class FSUtils {
return ((partitionPath == null) || (partitionPath.isEmpty())) ? basePath : new Path(basePath, partitionPath); return ((partitionPath == null) || (partitionPath.isEmpty())) ? basePath : new Path(basePath, partitionPath);
} }
/**
* Get DFS full partition path (e.g. hdfs://ip-address:8020:/<absolute path>)
*/
public static String getDFSFullPartitionPath(FileSystem fs, Path partitionPath) {
return fs.getUri() + partitionPath.toUri().getRawPath();
}
/** /**
* This is due to HUDI-140 GCS has a different behavior for detecting EOF during seek(). * This is due to HUDI-140 GCS has a different behavior for detecting EOF during seek().
* *

8
hudi-hive/src/main/java/org/apache/hudi/hive/HoodieHiveClient.java
View File

@@ -51,6 +51,7 @@ import org.apache.hudi.common.model.HoodieCommitMetadata;
import org.apache.hudi.common.model.HoodieFileFormat; import org.apache.hudi.common.model.HoodieFileFormat;
import org.apache.hudi.common.model.HoodieLogFile; import org.apache.hudi.common.model.HoodieLogFile;
import org.apache.hudi.common.model.HoodieTableType; import org.apache.hudi.common.model.HoodieTableType;
import org.apache.hudi.common.storage.StorageSchemes;
import org.apache.hudi.common.table.HoodieTableMetaClient; import org.apache.hudi.common.table.HoodieTableMetaClient;
import org.apache.hudi.common.table.HoodieTimeline; import org.apache.hudi.common.table.HoodieTimeline;
import org.apache.hudi.common.table.timeline.HoodieInstant; import org.apache.hudi.common.table.timeline.HoodieInstant;
@@ -194,7 +195,9 @@ public class HoodieHiveClient {
String alterTable = "ALTER TABLE " + syncConfig.tableName; String alterTable = "ALTER TABLE " + syncConfig.tableName;
for (String partition : partitions) { for (String partition : partitions) {
String partitionClause = getPartitionClause(partition); String partitionClause = getPartitionClause(partition);
String fullPartitionPath = FSUtils.getPartitionPath(syncConfig.basePath, partition).toString(); Path partitionPath = FSUtils.getPartitionPath(syncConfig.basePath, partition);
String fullPartitionPath = partitionPath.toUri().getScheme().equals(StorageSchemes.HDFS.getScheme())
? FSUtils.getDFSFullPartitionPath(fs, partitionPath) : partitionPath.toString();
String changePartition = String changePartition =
alterTable + " PARTITION (" + partitionClause + ") SET LOCATION '" + fullPartitionPath + "'"; alterTable + " PARTITION (" + partitionClause + ") SET LOCATION '" + fullPartitionPath + "'";
changePartitions.add(changePartition); changePartitions.add(changePartition);
@@ -218,7 +221,8 @@ public class HoodieHiveClient {
List<PartitionEvent> events = Lists.newArrayList(); List<PartitionEvent> events = Lists.newArrayList();
for (String storagePartition : partitionStoragePartitions) { for (String storagePartition : partitionStoragePartitions) {
String fullStoragePartitionPath = FSUtils.getPartitionPath(syncConfig.basePath, storagePartition).toString(); Path storagePartitionPath = FSUtils.getPartitionPath(syncConfig.basePath, storagePartition);
String fullStoragePartitionPath = Path.getPathWithoutSchemeAndAuthority(storagePartitionPath).toUri().getPath();
// Check if the partition values or if hdfs path is the same // Check if the partition values or if hdfs path is the same
List<String> storagePartitionValues = partitionValueExtractor.extractPartitionValuesInPath(storagePartition); List<String> storagePartitionValues = partitionValueExtractor.extractPartitionValuesInPath(storagePartition);
Collections.sort(storagePartitionValues); Collections.sort(storagePartitionValues);