lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-1919] Type mismatch when streaming read copy_on_write table using flink (#2986)

Browse Source 
* [HUDI-1919] Type mismatch when streaming read copy_on_write table using flink #2976

* Update ParquetSplitReaderUtil.java
This commit is contained in:
Town
2021-05-25 11:36:43 +08:00
committed by GitHub
parent 369a849337
commit aba1eadbfc
3 changed files with 108 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
hudi-flink/src/main/java/org/apache/hudi/table/format/cow/ParquetSplitReaderUtil.java
View File

@@ -95,16 +95,13 @@ public class ParquetSplitReaderUtil {
Path path,
long splitStart,
long splitLength) throws IOException {
List<String> nonPartNames = Arrays.stream(fullFieldNames)
List<String> selNonPartNames = Arrays.stream(selectedFields)
.mapToObj(i -> fullFieldNames[i])
.filter(n -> !partitionSpec.containsKey(n))
.collect(Collectors.toList());
List<String> selNonPartNames = Arrays.stream(selectedFields)
.mapToObj(i -> fullFieldNames[i])
.filter(nonPartNames::contains).collect(Collectors.toList());
int[] selParquetFields = selNonPartNames.stream()
.mapToInt(nonPartNames::indexOf)
int[] selParquetFields = Arrays.stream(selectedFields)
.filter(i -> !partitionSpec.containsKey(fullFieldNames[i]))
.toArray();
ParquetColumnarRowSplitReader.ColumnBatchGenerator gen = readVectors -> {