lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-4400] Fix missing bloom filters in metadata table in non-partitioned table (#6113)

Browse Source 
Fixes the missing bloom filters in metadata table in the non-partitioned table due to incorrect record key generation, because of wrong file names when generating the metadata payload for the bloom filter.
This commit is contained in:
Y Ethan Guo
2022-07-21 11:38:25 -07:00
committed by GitHub
parent f52b93fd10
commit 50cdb867c7
4 changed files with 190 additions and 50 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
hudi-common/src/main/java/org/apache/hudi/metadata/HoodieTableMetadataUtil.java
View File

@@ -55,6 +55,7 @@ import org.apache.hudi.exception.HoodieIOException;
import org.apache.hudi.exception.HoodieMetadataException;
import org.apache.hudi.io.storage.HoodieFileReader;
import org.apache.hudi.io.storage.HoodieFileReaderFactory;
import org.apache.hudi.util.Lazy;
import org.apache.avro.AvroTypeException;
import org.apache.avro.LogicalTypes;
@@ -63,7 +64,6 @@ import org.apache.avro.generic.GenericRecord;
import org.apache.avro.generic.IndexedRecord;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hudi.util.Lazy;
import org.apache.log4j.LogManager;
import org.apache.log4j.Logger;
@@ -409,8 +409,11 @@ public class HoodieTableMetadataUtil {
LOG.error("Failed to find path in write stat to update metadata table " + hoodieWriteStat);
return Collections.emptyListIterator();
}
int offset = partition.equals(NON_PARTITIONED_NAME) ? (pathWithPartition.startsWith("/") ? 1 : 0) :
partition.length() + 1;
// For partitioned table, "partition" contains the relative partition path;
// for non-partitioned table, "partition" is empty
int offset = StringUtils.isNullOrEmpty(partition)
? (pathWithPartition.startsWith("/") ? 1 : 0) : partition.length() + 1;
final String fileName = pathWithPartition.substring(offset);
if (!FSUtils.isBaseFile(new Path(fileName))) {