lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-4270] Bootstrap op data loading missing (#5888)

Browse Source
This commit is contained in:
Bo Cui
2022-06-21 11:47:39 +08:00
committed by GitHub
parent 5c204f1416
commit 7c4aaa9715
1 changed files with 2 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/bootstrap/BootstrapOperator.java
View File

@@ -22,6 +22,7 @@ import org.apache.hudi.common.fs.FSUtils;
import org.apache.hudi.common.model.FileSlice; import org.apache.hudi.common.model.FileSlice;
import org.apache.hudi.common.model.HoodieAvroRecord; import org.apache.hudi.common.model.HoodieAvroRecord;
import org.apache.hudi.common.model.HoodieKey; import org.apache.hudi.common.model.HoodieKey;
import org.apache.hudi.common.model.HoodieLogFile;
import org.apache.hudi.common.model.HoodieRecord; import org.apache.hudi.common.model.HoodieRecord;
import org.apache.hudi.common.model.HoodieRecordGlobalLocation; import org.apache.hudi.common.model.HoodieRecordGlobalLocation;
import org.apache.hudi.common.table.TableSchemaResolver; import org.apache.hudi.common.table.TableSchemaResolver;
@@ -223,6 +224,7 @@ public class BootstrapOperator<I, O extends HoodieRecord<?>>
// load avro log records // load avro log records
List<String> logPaths = fileSlice.getLogFiles() List<String> logPaths = fileSlice.getLogFiles()
.sorted(HoodieLogFile.getLogFileComparator())
// filter out crushed files // filter out crushed files
.filter(logFile -> isValidFile(logFile.getFileStatus())) .filter(logFile -> isValidFile(logFile.getFileStatus()))
.map(logFile -> logFile.getPath().toString()) .map(logFile -> logFile.getPath().toString())