lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-394] Provide a basic implementation of test suite

Browse Source
This commit is contained in:
Nishith Agarwal
2019-11-01 12:40:09 -07:00
committed by n3nash
parent d5b593b7d9
commit 2fc2b01d86
102 changed files with 8633 additions and 64 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/TestCsvDFSSource.java
View File

@@ -42,7 +42,7 @@ public class TestCsvDFSSource extends AbstractDFSSourceTestBase {
this.fileSuffix = ".json";
this.useFlattenedSchema = true;
this.schemaProvider = new FilebasedSchemaProvider(
Helpers.setupSchemaOnDFS("source-flattened.avsc"), jsc);
Helpers.setupSchemaOnDFS("delta-streamer-config", "source-flattened.avsc"), jsc);
}
@Override

35
hudi-utilities/src/test/java/org/apache/hudi/utilities/testutils/UtilitiesTestBase.java
View File

@@ -18,6 +18,7 @@
package org.apache.hudi.utilities.testutils;
import java.io.FileInputStream;
import org.apache.hudi.common.config.TypedProperties;
import org.apache.hudi.common.model.HoodieRecord;
import org.apache.hudi.common.model.HoodieTableType;
@@ -189,6 +190,17 @@ public class UtilitiesTestBase {
return sb.toString();
}
public static String readFileFromAbsolutePath(String absolutePathForResource) throws IOException {
BufferedReader reader =
new BufferedReader(new InputStreamReader(new FileInputStream(absolutePathForResource)));
StringBuffer sb = new StringBuffer();
String line;
while ((line = reader.readLine()) != null) {
sb.append(line + "\n");
}
return sb.toString();
}
public static void copyToDFS(String testResourcePath, FileSystem fs, String targetPath) throws IOException {
PrintStream os = new PrintStream(fs.create(new Path(targetPath), true));
os.print(readFile(testResourcePath));
@@ -196,6 +208,14 @@ public class UtilitiesTestBase {
os.close();
}
public static void copyToDFSFromAbsolutePath(String absolutePathForResource, FileSystem fs, String targetPath)
throws IOException {
PrintStream os = new PrintStream(fs.create(new Path(targetPath), true));
os.print(readFileFromAbsolutePath(absolutePathForResource));
os.flush();
os.close();
}
public static void savePropsToDFS(TypedProperties props, FileSystem fs, String targetPath) throws IOException {
String[] lines = props.keySet().stream().map(k -> String.format("%s=%s", k, props.get(k))).toArray(String[]::new);
saveStringsToDFS(lines, fs, targetPath);
@@ -258,11 +278,18 @@ public class UtilitiesTestBase {
}
public static TypedProperties setupSchemaOnDFS() throws IOException {
return setupSchemaOnDFS("source.avsc");
return setupSchemaOnDFS("delta-streamer-config", "source.avsc");
}
public static TypedProperties setupSchemaOnDFS(String filename) throws IOException {
UtilitiesTestBase.Helpers.copyToDFS("delta-streamer-config/" + filename, dfs, dfsBasePath + "/" + filename);
public static TypedProperties setupSchemaOnDFS(String scope, String filename) throws IOException {
UtilitiesTestBase.Helpers.copyToDFS(scope + "/" + filename, dfs, dfsBasePath + "/" + filename);
TypedProperties props = new TypedProperties();
props.setProperty("hoodie.deltastreamer.schemaprovider.source.schema.file", dfsBasePath + "/" + filename);
return props;
}
public static TypedProperties setupSchemaOnDFSWithAbsoluteScope(String scope, String filename) throws IOException {
UtilitiesTestBase.Helpers.copyToDFSFromAbsolutePath(scope + "/" + filename, dfs, dfsBasePath + "/" + filename);
TypedProperties props = new TypedProperties();
props.setProperty("hoodie.deltastreamer.schemaprovider.source.schema.file", dfsBasePath + "/" + filename);
return props;
@@ -278,7 +305,7 @@ public class UtilitiesTestBase {
}
public static List<GenericRecord> toGenericRecords(List<HoodieRecord> hoodieRecords) {
List<GenericRecord> records = new ArrayList<GenericRecord>();
List<GenericRecord> records = new ArrayList<>();
for (HoodieRecord hoodieRecord : hoodieRecords) {
records.add(toGenericRecord(hoodieRecord));
}

466
hudi-utilities/src/test/resources/delta-streamer-config/complex-source.avsc Normal file
View File

@@ -0,0 +1,466 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
{
"name": "COMPLEX",
"fields": [
{
"default": null,
"type": [
"null",
{
"items": "string",
"type": "array"
}
],
"name": "array_of_string_fields1"
},
{
"default": null,
"type": [
"null",
"string"
],
"name": "string_field1"
},
{
"default": null,
"type": [
"null",
"string"
],
"name": "string_field2"
},
{
"default": null,
"type": [
"null",
"string"
],
"name": "string_field3"
},
{
"default": null,
"type": [
"null",
"string"
],
"name": "string_field4"
},
{
"default": null,
"type": [
"null",
"string"
],
"name": "string_field5"
},
{
"default": null,
"type": [
"null",
"boolean"
],
"name": "boolean_field1"
},
{
"default": null,
"type": [
"null",
"string"
],
"name": "string_field6"
},
{
"default": null,
"type": [
"null",
"string"
],
"name": "string_field7"
},
{
"default": null,
"type": [
"null",
"string"
],
"name": "string_field8"
},
{
"default": null,
"type": [
"null",
"string"
],
"name": "string_field9"
},
{
"default": null,
"type": [
"null",
"string"
],
"name": "string_field10"
},
{
"default": null,
"type": [
"null",
"string"
],
"name": "string_field11"
},
{
"default": null,
"type": [
"null",
"string"
],
"name": "string_field12"
},
{
"default": null,
"type": [
"null",
"double"
],
"name": "double_field1"
},
{
"default": null,
"type": [
"null",
"string"
],
"name": "string_field13"
},
{
"default": null,
"type": [
"null",
"long"
],
"name": "long_field1"
},
{
"default": null,
"type": [
"null",
"string"
],
"name": "string_field14"
},
{
"default": null,
"type": [
"null",
"long"
],
"name": "long_field2"
},
{
"default": null,
"type": [
"null",
{
"items": {
"fields": [
{
"default": null,
"type": [
"null",
"string"
],
"name": "string_field15"
},
{
"default": null,
"type": [
"null",
"string"
],
"name": "string_field16"
},
{
"default": null,
"type": [
"null",
"string"
],
"name": "string_field17"
},
{
"default": null,
"type": [
"null",
"long"
],
"name": "long_field3"
},
{
"default": null,
"type": [
"null",
"long"
],
"name": "long_field4"
},
{
"default": null,
"type": [
"null",
"double"
],
"name": "double_field2"
},
{
"default": null,
"type": [
"null",
"double"
],
"name": "double_field3"
}
],
"type": "record",
"name": "record_field1"
},
"type": "array"
}
],
"name": "record_name1"
},
{
"default": null,
"type": [
"null",
"string"
],
"name": "string_field18"
},
{
"default": null,
"type": [
"null",
"long"
],
"name": "long_field5"
},
{
"default": null,
"type": [
"null",
"double"
],
"name": "double_field4"
},
{
"default": null,
"type": [
"null",
"double"
],
"name": "double_field5"
},
{
"default": null,
"type": [
"null",
"string"
],
"name": "string_field19"
},
{
"default": null,
"type": [
"null",
"long"
],
"name": "long_field6"
},
{
"default": null,
"type": [
"null",
"string"
],
"name": "string_field20"
},
{
"default": null,
"type": [
"null",
"long"
],
"name": "long_field7"
},
{
"default": null,
"type": [
"null",
"double"
],
"name": "double_field6"
},
{
"default": null,
"type": [
"null",
"string"
],
"name": "string_field21"
},
{
"default": null,
"type": [
"null",
"string"
],
"name": "string_field22"
},
{
"default": null,
"type": [
"null",
"string"
],
"name": "string_field23"
},
{
"default": null,
"type": [
"null",
"long"
],
"name": "long_field8"
},
{
"default": null,
"type": [
"null",
"double"
],
"name": "double_field7"
},
{
"default": null,
"type": [
"null",
"string"
],
"name": "string_field24"
},
{
"default": null,
"type": [
"null",
"long"
],
"name": "long_field10"
},
{
"default": null,
"type": [
"null",
"string"
],
"name": "string_field25"
},
{
"default": null,
"type": [
"null",
"string"
],
"name": "string_field26"
},
{
"default": null,
"type": [
"null",
"long"
],
"name": "long_field11"
},
{
"default": null,
"type": [
"null",
"boolean"
],
"name": "boolean_field3"
},
{
"default": null,
"type": [
"null",
"long"
],
"name": "long_field12"
},
{
"default": null,
"type": [
"null",
"double"
],
"name": "double_field8"
},
{
"default": null,
"type": [
"null",
"long"
],
"name": "long_field13"
},
{
"default": null,
"type": [
"null",
"string"
],
"name": "string_field27"
},
{
"default": null,
"type": [
"null",
"string"
],
"name": "string_field28"
},
{
"default": null,
"type": [
"null",
"string"
],
"name": "string_field29"
},
{
"default": null,
"type": [
"null",
"string"
],
"name": "string_field30"
}
],
"type": "record"
}