lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-1143] Change timestamp field in HoodieTestDataGenerator from double to long

Browse Source
This commit is contained in:
shenh062326
2020-09-06 16:00:45 +08:00
committed by n3nash
parent 6c84ef20ac
commit 581d54097c
18 changed files with 176 additions and 47 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
hudi-utilities/src/test/java/org/apache/hudi/utilities/functional/TestHDFSParquetImporter.java
View File

@@ -160,12 +160,12 @@ public class TestHDFSParquetImporter extends FunctionalTestHarness implements Se
List<Row> readData = ds.select("timestamp", "_row_key", "rider", "driver", "begin_lat", "begin_lon", "end_lat", "end_lon").collectAsList();
List<HoodieTripModel> result = readData.stream().map(row ->
new HoodieTripModel(row.getDouble(0), row.getString(1), row.getString(2), row.getString(3), row.getDouble(4),
new HoodieTripModel(row.getLong(0), row.getString(1), row.getString(2), row.getString(3), row.getDouble(4),
row.getDouble(5), row.getDouble(6), row.getDouble(7)))
.collect(Collectors.toList());
List<HoodieTripModel> expected = insertData.stream().map(g ->
new HoodieTripModel(Double.parseDouble(g.get("timestamp").toString()),
new HoodieTripModel(Long.parseLong(g.get("timestamp").toString()),
g.get("_row_key").toString(),
g.get("rider").toString(),
g.get("driver").toString(),
@@ -207,13 +207,13 @@ public class TestHDFSParquetImporter extends FunctionalTestHarness implements Se
List<Row> readData = ds.select("timestamp", "_row_key", "rider", "driver", "begin_lat", "begin_lon", "end_lat", "end_lon").collectAsList();
List<HoodieTripModel> result = readData.stream().map(row ->
new HoodieTripModel(row.getDouble(0), row.getString(1), row.getString(2), row.getString(3), row.getDouble(4),
new HoodieTripModel(row.getLong(0), row.getString(1), row.getString(2), row.getString(3), row.getDouble(4),
row.getDouble(5), row.getDouble(6), row.getDouble(7)))
.collect(Collectors.toList());
// get expected result.
List<HoodieTripModel> expected = expectData.stream().map(g ->
new HoodieTripModel(Double.parseDouble(g.get("timestamp").toString()),
new HoodieTripModel(Long.parseLong(g.get("timestamp").toString()),
g.get("_row_key").toString(),
g.get("rider").toString(),
g.get("driver").toString(),
@@ -336,7 +336,7 @@ public class TestHDFSParquetImporter extends FunctionalTestHarness implements Se
*/
public static class HoodieTripModel {
double timestamp;
long timestamp;
String rowKey;
String rider;
String driver;
@@ -345,7 +345,7 @@ public class TestHDFSParquetImporter extends FunctionalTestHarness implements Se
double endLat;
double endLon;
public HoodieTripModel(double timestamp, String rowKey, String rider, String driver, double beginLat,
public HoodieTripModel(long timestamp, String rowKey, String rider, String driver, double beginLat,
double beginLon, double endLat, double endLon) {
this.timestamp = timestamp;
this.rowKey = rowKey;

2
hudi-utilities/src/test/resources/delta-streamer-config/source-flattened.avsc
View File

@@ -21,7 +21,7 @@
"fields" : [
{
"name" : "timestamp",
"type" : "double"
"type" : "long"
}, {
"name" : "_row_key",
"type" : "string"

2
hudi-utilities/src/test/resources/delta-streamer-config/source.avsc
View File

@@ -21,7 +21,7 @@
"fields" : [
{
"name" : "timestamp",
"type" : "double"
"type" : "long"
}, {
"name" : "_row_key",
"type" : "string"

2
hudi-utilities/src/test/resources/delta-streamer-config/source_short_trip_uber.avsc
View File

@@ -21,7 +21,7 @@
"fields" : [
{
"name" : "timestamp",
"type" : "double"
"type" : "long"
}, {
"name" : "_row_key",
"type" : "string"

2
hudi-utilities/src/test/resources/delta-streamer-config/source_uber.avsc
View File

@@ -21,7 +21,7 @@
"fields" : [
{
"name" : "timestamp",
"type" : "double"
"type" : "long"
}, {
"name" : "_row_key",
"type" : "string"

2
hudi-utilities/src/test/resources/delta-streamer-config/target-flattened.avsc
View File

@@ -21,7 +21,7 @@
"fields" : [
{
"name" : "timestamp",
"type" : "double"
"type" : "long"
}, {
"name" : "_row_key",
"type" : "string"

2
hudi-utilities/src/test/resources/delta-streamer-config/target.avsc
View File

@@ -21,7 +21,7 @@
"fields" : [
{
"name" : "timestamp",
"type" : "double"
"type" : "long"
}, {
"name" : "_row_key",
"type" : "string"

2
hudi-utilities/src/test/resources/delta-streamer-config/target_short_trip_uber.avsc
View File

@@ -21,7 +21,7 @@
"fields" : [
{
"name" : "timestamp",
"type" : "double"
"type" : "long"
}, {
"name" : "_row_key",
"type" : "string"

2
hudi-utilities/src/test/resources/delta-streamer-config/target_uber.avsc
View File

@@ -21,7 +21,7 @@
"fields" : [
{
"name" : "timestamp",
"type" : "double"
"type" : "long"
}, {
"name" : "_row_key",
"type" : "string"