lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

Fix conversion of Spark struct type to Avro schema

Browse Source 
cr https://code.amazon.com/reviews/CR-17184364
This commit is contained in:
Mehrotra
2020-01-02 16:16:27 -08:00
committed by Bhavani Sudha Saktheeswaran
parent fd8f1c70c0
commit 2bb0c21a3d
7 changed files with 46 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
hudi-spark/src/test/java/DataSourceTestUtils.java
View File

@@ -34,7 +34,9 @@ public class DataSourceTestUtils {
try {
String str = ((TestRawTripPayload) record.getData()).getJsonData();
str = "{" + str.substring(str.indexOf("\"timestamp\":"));
return Option.of(str.replaceAll("}", ", \"partition\": \"" + record.getPartitionPath() + "\"}"));
// Remove the last } bracket
str = str.substring(0, str.length() - 1);
return Option.of(str + ", \"partition\": \"" + record.getPartitionPath() + "\"}");
} catch (IOException e) {
return Option.empty();
}

4
hudi-spark/src/test/java/HoodieJavaApp.java
View File

@@ -212,8 +212,8 @@ public class HoodieJavaApp {
.load(tablePath + (nonPartitionedTable ? "/*" : "/*/*/*/*"));
hoodieROViewDF.registerTempTable("hoodie_ro");
spark.sql("describe hoodie_ro").show();
// all trips whose fare was greater than 2.
spark.sql("select fare, begin_lon, begin_lat, timestamp from hoodie_ro where fare > 2.0").show();
// all trips whose fare amount was greater than 2.
spark.sql("select fare.amount, begin_lon, begin_lat, timestamp from hoodie_ro where fare.amount > 2.0").show();
if (tableType.equals(HoodieTableType.COPY_ON_WRITE.name())) {
/**

4
hudi-spark/src/test/java/HoodieJavaStreamingApp.java
View File

@@ -195,8 +195,8 @@ public class HoodieJavaStreamingApp {
.load(tablePath + "/*/*/*/*");
hoodieROViewDF.registerTempTable("hoodie_ro");
spark.sql("describe hoodie_ro").show();
// all trips whose fare was greater than 2.
spark.sql("select fare, begin_lon, begin_lat, timestamp from hoodie_ro where fare > 2.0").show();
// all trips whose fare amount was greater than 2.
spark.sql("select fare.amount, begin_lon, begin_lat, timestamp from hoodie_ro where fare.amount > 2.0").show();
if (tableType.equals(HoodieTableType.COPY_ON_WRITE.name())) {
/**