lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-1471] Make QuickStartUtils generate deletes according to specific ts (#2357)

Browse Source
This commit is contained in:
wangxianghu
2020-12-22 21:14:18 +08:00
committed by GitHub
parent 959afb8ba4
commit f8ccb2872d
1 changed files with 8 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
hudi-spark-datasource/hudi-spark/src/main/java/org/apache/hudi/QuickstartUtils.java
View File

@@ -193,8 +193,12 @@ public class QuickstartUtils {
* @return list of hoodie records to delete * @return list of hoodie records to delete
*/ */
public List<String> generateDeletes(List<Row> rows) { public List<String> generateDeletes(List<Row> rows) {
return rows.stream().map(row -> // if row.length() == 2, then the record contains "uuid" and "partitionpath" fields, otherwise,
convertToString(row.getAs("uuid"), row.getAs("partitionpath"))).filter(os -> os.isPresent()).map(os -> os.get()) // another field "ts" is available
return rows.stream().map(row -> row.length() == 2
? convertToString(row.getAs("uuid"), row.getAs("partitionpath"), null) :
convertToString(row.getAs("uuid"), row.getAs("partitionpath"), row.getAs("ts"))
).filter(os -> os.isPresent()).map(os -> os.get())
.collect(Collectors.toList()); .collect(Collectors.toList());
} }
@@ -215,10 +219,10 @@ public class QuickstartUtils {
} }
} }
private static Option<String> convertToString(String uuid, String partitionPath) { private static Option<String> convertToString(String uuid, String partitionPath, Long ts) {
StringBuffer stringBuffer = new StringBuffer(); StringBuffer stringBuffer = new StringBuffer();
stringBuffer.append("{"); stringBuffer.append("{");
stringBuffer.append("\"ts\": 0.0,"); stringBuffer.append("\"ts\": \"" + (ts == null ? "0.0" : ts) + "\",");
stringBuffer.append("\"uuid\": \"" + uuid + "\","); stringBuffer.append("\"uuid\": \"" + uuid + "\",");
stringBuffer.append("\"partitionpath\": \"" + partitionPath + "\""); stringBuffer.append("\"partitionpath\": \"" + partitionPath + "\"");
stringBuffer.append("}"); stringBuffer.append("}");