lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

Fixing delete util method

Browse Source
This commit is contained in:
Sivabalan Narayanan
2020-01-14 23:32:20 -05:00
committed by Bhavani Sudha Saktheeswaran
parent 87fdb769f0
commit 2b2f23aa60
1 changed files with 18 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

27
hudi-spark/src/main/java/org/apache/hudi/QuickstartUtils.java
View File

@@ -28,6 +28,7 @@ import org.apache.hudi.exception.HoodieIOException;
import org.apache.avro.Schema; import org.apache.avro.Schema;
import org.apache.avro.generic.GenericData; import org.apache.avro.generic.GenericData;
import org.apache.avro.generic.GenericRecord; import org.apache.avro.generic.GenericRecord;
import org.apache.spark.sql.Row;
import java.io.IOException; import java.io.IOException;
import java.util.ArrayList; import java.util.ArrayList;
@@ -171,19 +172,15 @@ public class QuickstartUtils {
} }
/** /**
* Generates new Deletes, randomly distributed across the keys above. There can be duplicates within the returned * Generates delete records for the passed in rows
* list
* *
* @param n Number of deletes (including dups) * @param rows List of {@link Row}s for which delete record need to be generated
* @return list of hoodie records to delete * @return list of hoodie records to delete
*/ */
public List<HoodieKey> generateDeletes(Integer n) throws IOException { public List<String> generateDeletes(List<Row> rows) {
List<HoodieKey> deletes = new ArrayList<>(); return rows.stream().map(row ->
for (int i = 0; i < n; i++) { convertToString(row.getAs("uuid"), row.getAs("partitionPath"))).filter(os -> os.isPresent()).map(os -> os.get())
HoodieKey key = existingKeys.get(rand.nextInt(numExistingKeys)); .collect(Collectors.toList());
deletes.add(key);
}
return deletes;
} }
public void close() { public void close() {
@@ -203,6 +200,16 @@ public class QuickstartUtils {
} }
} }
private static Option<String> convertToString(String uuid, String partitionPath) {
StringBuffer stringBuffer = new StringBuffer();
stringBuffer.append("{");
stringBuffer.append("\"ts\": 0.0,");
stringBuffer.append("\"uuid\": \"" + uuid + "\",");
stringBuffer.append("\"partitionpath\": \"" + partitionPath + "\"");
stringBuffer.append("}");
return Option.of(stringBuffer.toString());
}
public static List<String> convertToStringList(List<HoodieRecord> records) { public static List<String> convertToStringList(List<HoodieRecord> records) {
return records.stream().map(hr -> convertToString(hr)).filter(os -> os.isPresent()).map(os -> os.get()) return records.stream().map(hr -> convertToString(hr)).filter(os -> os.isPresent()).map(os -> os.get())
.collect(Collectors.toList()); .collect(Collectors.toList());