lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-3454] Fix partition name in all code paths for LogRecordScanner (#5252)

Browse Source 
* Depend on FSUtils#getRelativePartitionPath(basePath, logFilePath.getParent) 
to get the partition.

* If the list of log file paths in the split is empty, then fallback to usual behaviour.
This commit is contained in:
Sagar Sumit
2022-04-08 09:59:36 +05:30
committed by GitHub
parent 672974c412
commit df87095ef0
8 changed files with 92 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
hudi-cli/src/main/java/org/apache/hudi/cli/commands/HoodieLogFileCommand.java
View File

@@ -67,6 +67,9 @@ import java.util.stream.Collectors;
import scala.Tuple2;
import scala.Tuple3;
import static org.apache.hudi.common.fs.FSUtils.getRelativePartitionPath;
import static org.apache.hudi.common.util.ValidationUtils.checkArgument;
/**
* CLI command to display log file options.
*/
@@ -185,7 +188,7 @@ public class HoodieLogFileCommand implements CommandMarker {
.collect(Collectors.toList());
// logFilePaths size must > 1
assert logFilePaths.size() > 0 : "There is no log file";
checkArgument(logFilePaths.size() > 0, "There is no log file");
// TODO : readerSchema can change across blocks/log files, fix this inside Scanner
AvroSchemaConverter converter = new AvroSchemaConverter();
@@ -218,6 +221,7 @@ public class HoodieLogFileCommand implements CommandMarker {
.withSpillableMapBasePath(HoodieMemoryConfig.SPILLABLE_MAP_BASE_PATH.defaultValue())
.withDiskMapType(HoodieCommonConfig.SPILLABLE_DISK_MAP_TYPE.defaultValue())
.withBitCaskDiskMapCompressionEnabled(HoodieCommonConfig.DISK_MAP_BITCASK_COMPRESSION_ENABLED.defaultValue())
.withPartition(getRelativePartitionPath(new Path(client.getBasePath()), new Path(logFilePaths.get(0)).getParent()))
.build();
for (HoodieRecord<? extends HoodieRecordPayload> hoodieRecord : scanner) {
Option<IndexedRecord> record = hoodieRecord.getData().getInsertValue(readerSchema);