lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-902] Avoid exception when getSchemaProvider (#1584)

Browse Source 
* When no new input data, don't throw exception for null SchemaProvider
* Return the newly added NullSchemaProvider instead
This commit is contained in:
Raymond Xu
2020-05-15 21:33:02 -07:00
committed by GitHub
parent 25e0b75b3d
commit 2ada2ef50f
2 changed files with 59 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

24
hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/InputBatch.java
View File

@@ -18,10 +18,14 @@
package org.apache.hudi.utilities.sources;
import org.apache.hudi.common.config.TypedProperties;
import org.apache.hudi.common.util.Option;
import org.apache.hudi.exception.HoodieException;
import org.apache.hudi.utilities.schema.SchemaProvider;
import org.apache.avro.Schema;
import org.apache.spark.api.java.JavaSparkContext;
public class InputBatch<T> {
private final Option<T> batch;
@@ -49,9 +53,25 @@ public class InputBatch<T> {
}
public SchemaProvider getSchemaProvider() {
if (schemaProvider == null) {
if (batch.isPresent() && schemaProvider == null) {
throw new HoodieException("Please provide a valid schema provider class!");
}
return schemaProvider;
return Option.ofNullable(schemaProvider).orElse(new NullSchemaProvider());
}
public static class NullSchemaProvider extends SchemaProvider {
public NullSchemaProvider() {
this(null, null);
}
public NullSchemaProvider(TypedProperties props, JavaSparkContext jssc) {
super(props, jssc);
}
@Override
public Schema getSourceSchema() {
return Schema.create(Schema.Type.NULL);
}
}
}

37
hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/TestInputBatch.java Normal file
View File

@@ -0,0 +1,37 @@
package org.apache.hudi.utilities.sources;
import org.apache.hudi.common.util.Option;
import org.apache.hudi.exception.HoodieException;
import org.apache.hudi.utilities.schema.RowBasedSchemaProvider;
import org.apache.hudi.utilities.schema.SchemaProvider;
import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertSame;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
public class TestInputBatch {
@Test
public void getSchemaProviderShouldThrowException() {
final InputBatch<String> inputBatch = new InputBatch<>(Option.of("foo"), null, null);
Throwable t = assertThrows(HoodieException.class, inputBatch::getSchemaProvider);
assertEquals("Please provide a valid schema provider class!", t.getMessage());
}
@Test
public void getSchemaProviderShouldReturnNullSchemaProvider() {
final InputBatch<String> inputBatch = new InputBatch<>(Option.empty(), null, null);
SchemaProvider schemaProvider = inputBatch.getSchemaProvider();
assertTrue(schemaProvider instanceof InputBatch.NullSchemaProvider);
}
@Test
public void getSchemaProviderShouldReturnGivenSchemaProvider() {
SchemaProvider schemaProvider = new RowBasedSchemaProvider(null);
final InputBatch<String> inputBatch = new InputBatch<>(Option.of("foo"), null, schemaProvider);
assertSame(schemaProvider, inputBatch.getSchemaProvider());
}
}