lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-402]: code clean up in test cases

Browse Source
This commit is contained in:
Pratyaksh Sharma
2019-12-29 15:00:06 +05:30
committed by vinoth chandar
parent 98c0d8cf60
commit dde21e7315
62 changed files with 320 additions and 619 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
hudi-spark/src/main/java/org/apache/hudi/DataSourceUtils.java
View File

@@ -32,7 +32,6 @@ import org.apache.hudi.exception.DatasetNotFoundException;
import org.apache.hudi.exception.HoodieException;
import org.apache.hudi.exception.HoodieNotSupportedException;
import org.apache.hudi.hive.HiveSyncConfig;
import org.apache.hudi.hive.PartitionValueExtractor;
import org.apache.hudi.hive.SlashEncodedDayPartitionValueExtractor;
import org.apache.hudi.index.HoodieIndex;
@@ -69,7 +68,7 @@ public class DataSourceUtils {
*/
public static String getNestedFieldValAsString(GenericRecord record, String fieldName) {
Object obj = getNestedFieldVal(record, fieldName);
return (obj == null) ? null : obj.toString();
return obj.toString();
}
/**
@@ -119,17 +118,6 @@ public class DataSourceUtils {
}
}
/**
* Create a partition value extractor class via reflection, passing in any configs needed.
*/
public static PartitionValueExtractor createPartitionExtractor(String partitionExtractorClass) {
try {
return (PartitionValueExtractor) ReflectionUtils.loadClass(partitionExtractorClass);
} catch (Throwable e) {
throw new HoodieException("Could not load partition extractor class " + partitionExtractorClass, e);
}
}
/**
* Create a payload class via reflection, passing in an ordering/precombine value.
*/
@@ -152,7 +140,7 @@ public class DataSourceUtils {
}
public static HoodieWriteClient createHoodieClient(JavaSparkContext jssc, String schemaStr, String basePath,
String tblName, Map<String, String> parameters) throws Exception {
String tblName, Map<String, String> parameters) {
// inline compaction is on by default for MOR
boolean inlineCompact = parameters.get(DataSourceWriteOptions.STORAGE_TYPE_OPT_KEY())
@@ -198,7 +186,7 @@ public class DataSourceUtils {
@SuppressWarnings("unchecked")
public static JavaRDD<HoodieRecord> dropDuplicates(JavaSparkContext jssc, JavaRDD<HoodieRecord> incomingHoodieRecords,
HoodieWriteConfig writeConfig, Option<EmbeddedTimelineService> timelineService) throws Exception {
HoodieWriteConfig writeConfig, Option<EmbeddedTimelineService> timelineService) {
HoodieReadClient client = null;
try {
client = new HoodieReadClient<>(jssc, writeConfig, timelineService);
@@ -217,7 +205,7 @@ public class DataSourceUtils {
@SuppressWarnings("unchecked")
public static JavaRDD<HoodieRecord> dropDuplicates(JavaSparkContext jssc, JavaRDD<HoodieRecord> incomingHoodieRecords,
Map<String, String> parameters, Option<EmbeddedTimelineService> timelineService) throws Exception {
Map<String, String> parameters, Option<EmbeddedTimelineService> timelineService) {
HoodieWriteConfig writeConfig =
HoodieWriteConfig.newBuilder().withPath(parameters.get("path")).withProps(parameters).build();
return dropDuplicates(jssc, incomingHoodieRecords, writeConfig, timelineService);

2
hudi-spark/src/test/java/DataSourceTestUtils.java
View File

@@ -41,7 +41,7 @@ public class DataSourceTestUtils {
}
public static List<String> convertToStringList(List<HoodieRecord> records) {
return records.stream().map(hr -> convertToString(hr)).filter(os -> os.isPresent()).map(os -> os.get())
return records.stream().map(DataSourceTestUtils::convertToString).filter(Option::isPresent).map(Option::get)
.collect(Collectors.toList());
}

3
hudi-spark/src/test/java/HoodieJavaApp.java
View File

@@ -114,13 +114,12 @@ public class HoodieJavaApp {
} else {
dataGen = new HoodieTestDataGenerator();
}
List<HoodieRecord> recordsSoFar = new ArrayList<>();
/**
* Commit with only inserts
*/
// Generate some input..
recordsSoFar.addAll(dataGen.generateInserts("001"/* ignore */, 100));
List<HoodieRecord> recordsSoFar = new ArrayList<>(dataGen.generateInserts("001"/* ignore */, 100));
List<String> records1 = DataSourceTestUtils.convertToStringList(recordsSoFar);
Dataset<Row> inputDF1 = spark.read().json(jssc.parallelize(records1, 2));