lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-3920] Fix partition path construction in metadata table validator (#5365)

Browse Source
This commit is contained in:
Y Ethan Guo
2022-04-19 16:40:09 -07:00
committed by GitHub
parent 6f3fe880d2
commit 28fdddfee0
1 changed files with 4 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieMetadataTableValidator.java
View File

@@ -490,7 +490,8 @@ public class HoodieMetadataTableValidator implements Serializable {
// ignore partitions created by uncommitted ingestion. // ignore partitions created by uncommitted ingestion.
allPartitionPathsFromFS = allPartitionPathsFromFS.stream().parallel().filter(part -> { allPartitionPathsFromFS = allPartitionPathsFromFS.stream().parallel().filter(part -> {
HoodiePartitionMetadata hoodiePartitionMetadata = new HoodiePartitionMetadata(metaClient.getFs(), new Path(basePath, part)); HoodiePartitionMetadata hoodiePartitionMetadata =
new HoodiePartitionMetadata(metaClient.getFs(), FSUtils.getPartitionPath(basePath, part));
Option<String> instantOption = hoodiePartitionMetadata.readPartitionCreatedCommitTime(); Option<String> instantOption = hoodiePartitionMetadata.readPartitionCreatedCommitTime();
if (instantOption.isPresent()) { if (instantOption.isPresent()) {
@@ -983,7 +984,7 @@ public class HoodieMetadataTableValidator implements Serializable {
return baseFileNameList.stream().flatMap(filename -> return baseFileNameList.stream().flatMap(filename ->
new ParquetUtils().readRangeFromParquetMetadata( new ParquetUtils().readRangeFromParquetMetadata(
metaClient.getHadoopConf(), metaClient.getHadoopConf(),
new Path(new Path(metaClient.getBasePath(), partitionPath), filename), new Path(FSUtils.getPartitionPath(metaClient.getBasePath(), partitionPath), filename),
allColumnNameList).stream()) allColumnNameList).stream())
.sorted(new HoodieColumnRangeMetadataComparator()) .sorted(new HoodieColumnRangeMetadataComparator())
.collect(Collectors.toList()); .collect(Collectors.toList());
@@ -1024,7 +1025,7 @@ public class HoodieMetadataTableValidator implements Serializable {
} }
private Option<BloomFilterData> readBloomFilterFromFile(String partitionPath, String filename) { private Option<BloomFilterData> readBloomFilterFromFile(String partitionPath, String filename) {
Path path = new Path(new Path(metaClient.getBasePath(), partitionPath), filename); Path path = new Path(FSUtils.getPartitionPath(metaClient.getBasePath(), partitionPath), filename);
HoodieFileReader<IndexedRecord> fileReader; HoodieFileReader<IndexedRecord> fileReader;
try { try {
fileReader = HoodieFileReaderFactory.getFileReader(metaClient.getHadoopConf(), path); fileReader = HoodieFileReaderFactory.getFileReader(metaClient.getHadoopConf(), path);