lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-995] Use Transformations, Assertions and SchemaTestUtil (#1884)

Browse Source 
- Consolidate transform functions for tests in Transformations.java
- Consolidate assertion functions for tests in Assertions.java
- Make use of SchemaTestUtil for loading schema from resource
This commit is contained in:
Raymond Xu
2020-08-01 05:57:18 -07:00
committed by GitHub
parent e79fbc07fe
commit 10e4268792
23 changed files with 302 additions and 277 deletions
Download Patch File Download Diff File Expand all files Collapse all files

35
hudi-common/src/test/java/org/apache/hudi/common/table/timeline/TestHoodieActiveTimeline.java
View File

@@ -22,7 +22,6 @@ import org.apache.hudi.common.table.HoodieTableMetaClient;
import org.apache.hudi.common.table.timeline.HoodieInstant.State;
import org.apache.hudi.common.table.timeline.versioning.TimelineLayoutVersion;
import org.apache.hudi.common.testutils.HoodieCommonTestHarness;
import org.apache.hudi.common.testutils.HoodieTestUtils;
import org.apache.hudi.common.testutils.MockHoodieTimeline;
import org.apache.hudi.common.util.CollectionUtils;
import org.apache.hudi.common.util.Option;
@@ -44,6 +43,7 @@ import java.util.stream.Collectors;
import java.util.stream.Stream;
import static org.apache.hudi.common.table.timeline.versioning.TimelineLayoutVersion.VERSION_0;
import static org.apache.hudi.common.testutils.Assertions.assertStreamEquals;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
@@ -94,17 +94,19 @@ public class TestHoodieActiveTimeline extends HoodieCommonTestHarness {
timeline = timeline.reload();
assertEquals(5, timeline.countInstants(), "Total instants should be 5");
HoodieTestUtils.assertStreamEquals("Check the instants stream",
assertStreamEquals(
Stream.of(instant1Complete, instant2Complete, instant3Complete, instant4Complete, instant5),
timeline.getInstants());
HoodieTestUtils.assertStreamEquals("Check the instants stream",
timeline.getInstants(), "Check the instants stream");
assertStreamEquals(
Stream.of(instant1Complete, instant2Complete, instant3Complete, instant4Complete, instant5),
timeline.getCommitTimeline().getInstants());
HoodieTestUtils.assertStreamEquals("Check the instants stream",
timeline.getCommitTimeline().getInstants(), "Check the instants stream");
assertStreamEquals(
Stream.of(instant1Complete, instant2Complete, instant3Complete, instant4Complete),
timeline.getCommitTimeline().filterCompletedInstants().getInstants());
HoodieTestUtils.assertStreamEquals("Check the instants stream", Stream.of(instant5),
timeline.getCommitTimeline().filterPendingExcludingCompaction().getInstants());
timeline.getCommitTimeline().filterCompletedInstants().getInstants(),
"Check the instants stream");
assertStreamEquals(Stream.of(instant5),
timeline.getCommitTimeline().filterPendingExcludingCompaction().getInstants(),
"Check the instants stream");
// Backwards compatibility testing for reading compaction plans
metaClient = HoodieTableMetaClient.initTableType(metaClient.getHadoopConf(),
@@ -150,15 +152,18 @@ public class TestHoodieActiveTimeline extends HoodieCommonTestHarness {
public void testTimelineOperations() {
timeline = new MockHoodieTimeline(Stream.of("01", "03", "05", "07", "09", "11", "13", "15", "17", "19"),
Stream.of("21", "23"));
HoodieTestUtils.assertStreamEquals("findInstantsInRange should return 4 instants", Stream.of("05", "07", "09", "11"),
assertStreamEquals(Stream.of("05", "07", "09", "11"),
timeline.getCommitTimeline().filterCompletedInstants().findInstantsInRange("04", "11")
.getInstants().map(HoodieInstant::getTimestamp));
HoodieTestUtils.assertStreamEquals("findInstantsAfter 07 should return 2 instants", Stream.of("09", "11"),
.getInstants().map(HoodieInstant::getTimestamp),
"findInstantsInRange should return 4 instants");
assertStreamEquals(Stream.of("09", "11"),
timeline.getCommitTimeline().filterCompletedInstants().findInstantsAfter("07", 2)
.getInstants().map(HoodieInstant::getTimestamp));
HoodieTestUtils.assertStreamEquals("findInstantsBefore 07 should return 3 instants", Stream.of("01", "03", "05"),
.getInstants().map(HoodieInstant::getTimestamp),
"findInstantsAfter 07 should return 2 instants");
assertStreamEquals(Stream.of("01", "03", "05"),
timeline.getCommitTimeline().filterCompletedInstants().findInstantsBefore("07")
.getInstants().map(HoodieInstant::getTimestamp));
.getInstants().map(HoodieInstant::getTimestamp),
"findInstantsBefore 07 should return 3 instants");
assertFalse(timeline.empty());
assertFalse(timeline.getCommitTimeline().filterPendingExcludingCompaction().empty());
assertEquals(12, timeline.countInstants());

41
hudi-common/src/test/java/org/apache/hudi/common/testutils/Assertions.java Normal file
View File

@@ -0,0 +1,41 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.hudi.common.testutils;
import java.util.Iterator;
import java.util.stream.Stream;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
/**
* Common assertion functions.
*/
public class Assertions {
public static void assertStreamEquals(Stream<?> expected, Stream<?> actual, String message) {
Iterator<?> iter1 = expected.iterator();
Iterator<?> iter2 = actual.iterator();
while (iter1.hasNext() && iter2.hasNext()) {
assertEquals(iter1.next(), iter2.next(), message);
}
assertTrue(!iter1.hasNext() && !iter2.hasNext(), message);
}
}

11
hudi-common/src/test/java/org/apache/hudi/common/testutils/HoodieTestUtils.java
View File

@@ -83,7 +83,6 @@ import java.util.Calendar;
import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Properties;
@@ -93,7 +92,6 @@ import java.util.stream.Collectors;
import java.util.stream.Stream;
import static org.apache.hudi.common.table.timeline.HoodieActiveTimeline.COMMIT_FORMATTER;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.fail;
/**
@@ -388,15 +386,6 @@ public class HoodieTestUtils {
}
}
public static void assertStreamEquals(String message, Stream<?> expected, Stream<?> actual) {
Iterator<?> iter1 = expected.iterator();
Iterator<?> iter2 = actual.iterator();
while (iter1.hasNext() && iter2.hasNext()) {
assertEquals(iter1.next(), iter2.next(), message);
}
assert !iter1.hasNext() && !iter2.hasNext();
}
public static <T extends Serializable> T serializeDeserialize(T object, Class<T> clazz) {
// Using Kyro as the default serializer in Spark Jobs
Kryo kryo = new Kryo();

20
hudi-common/src/test/java/org/apache/hudi/common/testutils/RawTripTestPayload.java
View File

@@ -20,6 +20,7 @@
package org.apache.hudi.common.testutils;
import org.apache.hudi.avro.MercifulJsonConverter;
import org.apache.hudi.common.model.HoodieRecord;
import org.apache.hudi.common.model.HoodieRecordPayload;
import org.apache.hudi.common.util.FileIOUtils;
import org.apache.hudi.common.util.Option;
@@ -32,7 +33,9 @@ import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import java.util.zip.Deflater;
import java.util.zip.DeflaterOutputStream;
import java.util.zip.InflaterInputStream;
@@ -74,6 +77,23 @@ public class RawTripTestPayload implements HoodieRecordPayload<RawTripTestPayloa
this.isDeleted = false;
}
public static List<String> recordsToStrings(List<HoodieRecord> records) {
return records.stream().map(RawTripTestPayload::recordToString).filter(Option::isPresent).map(Option::get)
.collect(Collectors.toList());
}
public static Option<String> recordToString(HoodieRecord record) {
try {
String str = ((RawTripTestPayload) record.getData()).getJsonData();
str = "{" + str.substring(str.indexOf("\"timestamp\":"));
// Remove the last } bracket
str = str.substring(0, str.length() - 1);
return Option.of(str + ", \"partition\": \"" + record.getPartitionPath() + "\"}");
} catch (IOException e) {
return Option.empty();
}
}
public String getPartitionPath() {
return partitionPath;
}

25
hudi-common/src/test/java/org/apache/hudi/common/testutils/SchemaTestUtil.java
View File

@@ -52,7 +52,9 @@ import java.util.stream.Stream;
/**
* A utility class for testing schema.
*/
public class SchemaTestUtil {
public final class SchemaTestUtil {
private static final String RESOURCE_SAMPLE_DATA = "/sample.data";
public static Schema getSimpleSchema() throws IOException {
return new Schema.Parser().parse(SchemaTestUtil.class.getResourceAsStream("/simple-test.avsc"));
@@ -66,12 +68,12 @@ public class SchemaTestUtil {
throws IOException, URISyntaxException {
GenericDatumReader<IndexedRecord> reader = new GenericDatumReader<>(writerSchema, readerSchema);
// Required to register the necessary JAR:// file system
URI resource = SchemaTestUtil.class.getClass().getResource("/sample.data").toURI();
URI resource = SchemaTestUtil.class.getResource(RESOURCE_SAMPLE_DATA).toURI();
Path dataPath;
if (resource.toString().contains("!")) {
dataPath = uriToPath(resource);
} else {
dataPath = Paths.get(SchemaTestUtil.class.getClass().getResource("/sample.data").toURI());
dataPath = Paths.get(SchemaTestUtil.class.getResource(RESOURCE_SAMPLE_DATA).toURI());
}
try (Stream<String> stream = Files.lines(dataPath)) {
@@ -79,11 +81,11 @@ public class SchemaTestUtil {
try {
return reader.read(null, DecoderFactory.get().jsonDecoder(writerSchema, s));
} catch (IOException e) {
throw new HoodieIOException("Could not read data from simple_data.json", e);
throw new HoodieIOException("Could not read data from " + RESOURCE_SAMPLE_DATA, e);
}
}).collect(Collectors.toList());
} catch (IOException e) {
throw new HoodieIOException("Could not read data from simple_data.json", e);
throw new HoodieIOException("Could not read data from " + RESOURCE_SAMPLE_DATA, e);
}
}
@@ -181,4 +183,17 @@ public class SchemaTestUtil {
MercifulJsonConverter converter = new MercifulJsonConverter();
return converter.convert(record.toJsonString(), schema);
}
public static Schema getSchemaFromResource(Class<?> clazz, String name, boolean withHoodieMetadata) {
try {
Schema schema = new Schema.Parser().parse(clazz.getResourceAsStream(name));
return withHoodieMetadata ? HoodieAvroUtils.addMetadataFields(schema) : schema;
} catch (IOException e) {
throw new RuntimeException(String.format("Failed to get schema from resource `%s` for class `%s`", name, clazz.getName()));
}
}
public static Schema getSchemaFromResource(Class<?> clazz, String name) {
return getSchemaFromResource(clazz, name, false);
}
}

75
hudi-common/src/test/java/org/apache/hudi/common/testutils/Transformations.java Normal file
View File

@@ -0,0 +1,75 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.hudi.common.testutils;
import org.apache.hudi.common.model.HoodieKey;
import org.apache.hudi.common.model.HoodieRecord;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
/**
* Common transformations in test cases.
*/
public final class Transformations {
public static <T> List<T> flatten(Iterator<List<T>> iteratorOfLists) {
List<T> flattened = new ArrayList<>();
iteratorOfLists.forEachRemaining(flattened::addAll);
return flattened;
}
public static <T> Iterator<T> flattenAsIterator(Iterator<List<T>> iteratorOfLists) {
return flatten(iteratorOfLists).iterator();
}
public static Set<String> recordsToRecordKeySet(List<HoodieRecord> records) {
return records.stream().map(HoodieRecord::getRecordKey).collect(Collectors.toSet());
}
public static List<HoodieKey> recordsToHoodieKeys(List<HoodieRecord> records) {
return records.stream().map(HoodieRecord::getKey).collect(Collectors.toList());
}
/**
* Pseudorandom: select even indices first, then select odd ones.
*/
public static <T> List<T> randomSelect(List<T> items, int n) {
int s = items.size();
if (n < 0 || n > s) {
throw new IllegalArgumentException(String.format("Invalid number of items to select! Valid range for n: [0, %s]", s));
}
List<T> selected = new ArrayList<>();
for (int i = 0, numSelected = 0; i < s && numSelected < n; i += 2, numSelected++) {
selected.add(items.get(i));
}
for (int i = 1, numSelected = selected.size(); i < s && numSelected < n; i += 2, numSelected++) {
selected.add(items.get(i));
}
return selected;
}
public static List<HoodieKey> randomSelectAsHoodieKeys(List<HoodieRecord> records, int n) {
return randomSelect(recordsToHoodieKeys(records), n);
}
}