lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-2134]Add generics to avoif forced conversion in BaseSparkCommitActionExecutor#partition (#3232)

Browse Source
This commit is contained in:
Yungthuis
2021-07-08 13:31:38 +08:00
committed by GitHub
parent 16e90d30ea
commit 1d3cd06572
1 changed files with 5 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/commit/BaseSparkCommitActionExecutor.java
View File

@@ -188,16 +188,16 @@ public abstract class BaseSparkCommitActionExecutor<T extends HoodieRecordPayloa
} }
private JavaRDD<HoodieRecord<T>> partition(JavaRDD<HoodieRecord<T>> dedupedRecords, Partitioner partitioner) { private JavaRDD<HoodieRecord<T>> partition(JavaRDD<HoodieRecord<T>> dedupedRecords, Partitioner partitioner) {
JavaPairRDD<Tuple2, HoodieRecord<T>> mappedRDD = dedupedRecords.mapToPair( JavaPairRDD<Tuple2<HoodieKey, Option<HoodieRecordLocation>>, HoodieRecord<T>> mappedRDD = dedupedRecords.mapToPair(
record -> new Tuple2<>(new Tuple2<>(record.getKey(), Option.ofNullable(record.getCurrentLocation())), record)); record -> new Tuple2<>(new Tuple2<>(record.getKey(), Option.ofNullable(record.getCurrentLocation())), record));
JavaPairRDD<Tuple2, HoodieRecord<T>> partitionedRDD; JavaPairRDD<Tuple2<HoodieKey, Option<HoodieRecordLocation>>, HoodieRecord<T>> partitionedRDD;
if (table.requireSortedRecords()) { if (table.requireSortedRecords()) {
// Partition and sort within each partition as a single step. This is faster than partitioning first and then // Partition and sort within each partition as a single step. This is faster than partitioning first and then
// applying a sort. // applying a sort.
Comparator<Tuple2> comparator = (Comparator<Tuple2> & Serializable)(t1, t2) -> { Comparator<Tuple2<HoodieKey, Option<HoodieRecordLocation>>> comparator = (Comparator<Tuple2<HoodieKey, Option<HoodieRecordLocation>>> & Serializable)(t1, t2) -> {
HoodieKey key1 = (HoodieKey) t1._1; HoodieKey key1 = t1._1;
HoodieKey key2 = (HoodieKey) t2._1; HoodieKey key2 = t2._1;
return key1.getRecordKey().compareTo(key2.getRecordKey()); return key1.getRecordKey().compareTo(key2.getRecordKey());
}; };