lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

Fixed partitions produced by layout optimization in case order-by key is composed of a single column (#4183)

Browse Source
This commit is contained in:
Alexey Kudinkin
2021-12-01 20:56:04 -08:00
committed by GitHub
parent 5284730175
commit 772f5ca24e
1 changed files with 1 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
hudi-client/hudi-spark-client/src/main/java/org/apache/spark/OrderingIndexHelper.java
View File

@@ -100,7 +100,7 @@ public class OrderingIndexHelper {
} }
// only one col to sort, no need to use z-order // only one col to sort, no need to use z-order
if (sortCols.size() == 1) { if (sortCols.size() == 1) {
return df.repartitionByRange(fieldNum, org.apache.spark.sql.functions.col(sortCols.get(0))); return df.repartitionByRange(fileNum, org.apache.spark.sql.functions.col(sortCols.get(0)));
} }
Map<Integer, StructField> fieldMap = sortCols Map<Integer, StructField> fieldMap = sortCols
.stream().collect(Collectors.toMap(e -> Arrays.asList(df.schema().fields()).indexOf(columnsMap.get(e)), e -> columnsMap.get(e))); .stream().collect(Collectors.toMap(e -> Arrays.asList(df.schema().fields()).indexOf(columnsMap.get(e)), e -> columnsMap.get(e)));