[HUDI-1069] Remove duplicate assertNoWriteErrors() (#1797)
This commit is contained in:
@@ -42,6 +42,7 @@ import java.io.File;
|
|||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.stream.Collectors;
|
import java.util.stream.Collectors;
|
||||||
|
|
||||||
|
import static org.apache.hudi.testutils.Assertions.assertNoWriteErrors;
|
||||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||||
|
|||||||
@@ -72,6 +72,7 @@ import java.util.stream.Collectors;
|
|||||||
|
|
||||||
import static org.apache.hudi.common.table.timeline.versioning.TimelineLayoutVersion.VERSION_0;
|
import static org.apache.hudi.common.table.timeline.versioning.TimelineLayoutVersion.VERSION_0;
|
||||||
import static org.apache.hudi.common.util.ParquetUtils.readRowKeysFromParquet;
|
import static org.apache.hudi.common.util.ParquetUtils.readRowKeysFromParquet;
|
||||||
|
import static org.apache.hudi.testutils.Assertions.assertNoWriteErrors;
|
||||||
import static org.apache.hudi.testutils.HoodieTestDataGenerator.NULL_SCHEMA;
|
import static org.apache.hudi.testutils.HoodieTestDataGenerator.NULL_SCHEMA;
|
||||||
import static org.apache.hudi.testutils.HoodieTestDataGenerator.TRIP_EXAMPLE_SCHEMA;
|
import static org.apache.hudi.testutils.HoodieTestDataGenerator.TRIP_EXAMPLE_SCHEMA;
|
||||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||||
|
|||||||
@@ -37,6 +37,7 @@ import java.util.Collection;
|
|||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.stream.Collectors;
|
import java.util.stream.Collectors;
|
||||||
|
|
||||||
|
import static org.apache.hudi.testutils.Assertions.assertNoWriteErrors;
|
||||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||||
|
|
||||||
|
|||||||
@@ -73,6 +73,7 @@ import java.util.UUID;
|
|||||||
|
|
||||||
import scala.Tuple2;
|
import scala.Tuple2;
|
||||||
|
|
||||||
|
import static org.apache.hudi.testutils.Assertions.assertNoWriteErrors;
|
||||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||||
@@ -544,13 +545,6 @@ public class TestHoodieIndex extends HoodieClientTestHarness {
|
|||||||
this.index = writeClient.getIndex();
|
this.index = writeClient.getIndex();
|
||||||
}
|
}
|
||||||
|
|
||||||
private void assertNoWriteErrors(List<WriteStatus> statuses) {
|
|
||||||
// Verify there are no errors
|
|
||||||
for (WriteStatus status : statuses) {
|
|
||||||
assertFalse(status.hasErrors());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
public static class DummyHoodieIndex<T extends HoodieRecordPayload> extends HoodieIndex<T> {
|
public static class DummyHoodieIndex<T extends HoodieRecordPayload> extends HoodieIndex<T> {
|
||||||
|
|
||||||
public DummyHoodieIndex(HoodieWriteConfig config) {
|
public DummyHoodieIndex(HoodieWriteConfig config) {
|
||||||
|
|||||||
@@ -62,8 +62,8 @@ import java.util.List;
|
|||||||
|
|
||||||
import scala.Tuple2;
|
import scala.Tuple2;
|
||||||
|
|
||||||
|
import static org.apache.hudi.testutils.Assertions.assertNoWriteErrors;
|
||||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
|
||||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||||
import static org.mockito.ArgumentMatchers.any;
|
import static org.mockito.ArgumentMatchers.any;
|
||||||
@@ -510,13 +510,6 @@ public class TestHBaseIndex extends HoodieClientTestHarness {
|
|||||||
return writeStatus;
|
return writeStatus;
|
||||||
}
|
}
|
||||||
|
|
||||||
private void assertNoWriteErrors(List<WriteStatus> statuses) {
|
|
||||||
// Verify there are no errors
|
|
||||||
for (WriteStatus status : statuses) {
|
|
||||||
assertFalse(status.hasErrors(), "Errors found in write of " + status.getFileId());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private HoodieWriteConfig getConfig() {
|
private HoodieWriteConfig getConfig() {
|
||||||
return getConfigBuilder(100).build();
|
return getConfigBuilder(100).build();
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -47,8 +47,8 @@ import java.nio.file.Paths;
|
|||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
|
|
||||||
|
import static org.apache.hudi.testutils.Assertions.assertNoWriteErrors;
|
||||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
|
||||||
import static org.junit.jupiter.api.Assertions.assertNotEquals;
|
import static org.junit.jupiter.api.Assertions.assertNotEquals;
|
||||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||||
|
|
||||||
@@ -306,18 +306,6 @@ public class TestHoodieMergeHandle extends HoodieClientTestHarness {
|
|||||||
return dataSet;
|
return dataSet;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* Assert no failures in writing hoodie files.
|
|
||||||
*
|
|
||||||
* @param statuses List of Write Status
|
|
||||||
*/
|
|
||||||
void assertNoWriteErrors(List<WriteStatus> statuses) {
|
|
||||||
// Verify there are no errors
|
|
||||||
for (WriteStatus status : statuses) {
|
|
||||||
assertFalse(status.hasErrors(), "Errors found in write of " + status.getFileId());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
HoodieWriteConfig.Builder getConfigBuilder() {
|
HoodieWriteConfig.Builder getConfigBuilder() {
|
||||||
return HoodieWriteConfig.newBuilder().withPath(basePath).withSchema(HoodieTestDataGenerator.TRIP_EXAMPLE_SCHEMA)
|
return HoodieWriteConfig.newBuilder().withPath(basePath).withSchema(HoodieTestDataGenerator.TRIP_EXAMPLE_SCHEMA)
|
||||||
.withParallelism(2, 2)
|
.withParallelism(2, 2)
|
||||||
|
|||||||
@@ -82,6 +82,7 @@ import java.util.stream.Stream;
|
|||||||
import scala.Tuple3;
|
import scala.Tuple3;
|
||||||
|
|
||||||
import static org.apache.hudi.common.testutils.HoodieTestUtils.DEFAULT_PARTITION_PATHS;
|
import static org.apache.hudi.common.testutils.HoodieTestUtils.DEFAULT_PARTITION_PATHS;
|
||||||
|
import static org.apache.hudi.testutils.Assertions.assertNoWriteErrors;
|
||||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||||
import static org.junit.jupiter.api.Assertions.assertNull;
|
import static org.junit.jupiter.api.Assertions.assertNull;
|
||||||
|
|||||||
@@ -47,10 +47,10 @@ import org.apache.hudi.config.HoodieIndexConfig;
|
|||||||
import org.apache.hudi.config.HoodieStorageConfig;
|
import org.apache.hudi.config.HoodieStorageConfig;
|
||||||
import org.apache.hudi.config.HoodieWriteConfig;
|
import org.apache.hudi.config.HoodieWriteConfig;
|
||||||
import org.apache.hudi.exception.HoodieIOException;
|
import org.apache.hudi.exception.HoodieIOException;
|
||||||
import org.apache.hudi.hadoop.utils.HoodieHiveUtils;
|
|
||||||
import org.apache.hudi.hadoop.utils.HoodieInputFormatUtils;
|
|
||||||
import org.apache.hudi.hadoop.HoodieParquetInputFormat;
|
import org.apache.hudi.hadoop.HoodieParquetInputFormat;
|
||||||
import org.apache.hudi.hadoop.realtime.HoodieParquetRealtimeInputFormat;
|
import org.apache.hudi.hadoop.realtime.HoodieParquetRealtimeInputFormat;
|
||||||
|
import org.apache.hudi.hadoop.utils.HoodieHiveUtils;
|
||||||
|
import org.apache.hudi.hadoop.utils.HoodieInputFormatUtils;
|
||||||
import org.apache.hudi.index.HoodieIndex;
|
import org.apache.hudi.index.HoodieIndex;
|
||||||
import org.apache.hudi.index.HoodieIndex.IndexType;
|
import org.apache.hudi.index.HoodieIndex.IndexType;
|
||||||
import org.apache.hudi.table.action.deltacommit.DeleteDeltaCommitActionExecutor;
|
import org.apache.hudi.table.action.deltacommit.DeleteDeltaCommitActionExecutor;
|
||||||
@@ -86,6 +86,7 @@ import java.util.Set;
|
|||||||
import java.util.stream.Collectors;
|
import java.util.stream.Collectors;
|
||||||
import java.util.stream.Stream;
|
import java.util.stream.Stream;
|
||||||
|
|
||||||
|
import static org.apache.hudi.testutils.Assertions.assertNoWriteErrors;
|
||||||
import static org.apache.hudi.testutils.HoodieTestDataGenerator.TRIP_EXAMPLE_SCHEMA;
|
import static org.apache.hudi.testutils.HoodieTestDataGenerator.TRIP_EXAMPLE_SCHEMA;
|
||||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||||
@@ -1400,13 +1401,6 @@ public class TestHoodieMergeOnReadTable extends HoodieClientTestHarness {
|
|||||||
.withIndexConfig(HoodieIndexConfig.newBuilder().withIndexType(indexType).build());
|
.withIndexConfig(HoodieIndexConfig.newBuilder().withIndexType(indexType).build());
|
||||||
}
|
}
|
||||||
|
|
||||||
private void assertNoWriteErrors(List<WriteStatus> statuses) {
|
|
||||||
// Verify there are no errors
|
|
||||||
for (WriteStatus status : statuses) {
|
|
||||||
assertFalse(status.hasErrors(), "Errors found in write of " + status.getFileId());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private FileStatus[] insertAndGetFilePaths(List<HoodieRecord> records, HoodieWriteClient client,
|
private FileStatus[] insertAndGetFilePaths(List<HoodieRecord> records, HoodieWriteClient client,
|
||||||
HoodieWriteConfig cfg, String commitTime) throws IOException {
|
HoodieWriteConfig cfg, String commitTime) throws IOException {
|
||||||
JavaRDD<HoodieRecord> writeRecords = jsc.parallelize(records, 1);
|
JavaRDD<HoodieRecord> writeRecords = jsc.parallelize(records, 1);
|
||||||
|
|||||||
@@ -61,6 +61,7 @@ import java.util.List;
|
|||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
import java.util.stream.Collectors;
|
import java.util.stream.Collectors;
|
||||||
|
|
||||||
|
import static org.apache.hudi.testutils.Assertions.assertNoWriteErrors;
|
||||||
import static org.apache.hudi.testutils.HoodieTestDataGenerator.TRIP_EXAMPLE_SCHEMA;
|
import static org.apache.hudi.testutils.HoodieTestDataGenerator.TRIP_EXAMPLE_SCHEMA;
|
||||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||||
|
|||||||
@@ -0,0 +1,41 @@
|
|||||||
|
/*
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
* or more contributor license agreements. See the NOTICE file
|
||||||
|
* distributed with this work for additional information
|
||||||
|
* regarding copyright ownership. The ASF licenses this file
|
||||||
|
* to you under the Apache License, Version 2.0 (the
|
||||||
|
* "License"); you may not use this file except in compliance
|
||||||
|
* with the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing,
|
||||||
|
* software distributed under the License is distributed on an
|
||||||
|
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||||
|
* KIND, either express or implied. See the License for the
|
||||||
|
* specific language governing permissions and limitations
|
||||||
|
* under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package org.apache.hudi.testutils;
|
||||||
|
|
||||||
|
import org.apache.hudi.client.WriteStatus;
|
||||||
|
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertAll;
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Commonly used assertion functions.
|
||||||
|
*/
|
||||||
|
public class Assertions {
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Assert no failures in writing hoodie files.
|
||||||
|
*/
|
||||||
|
public static void assertNoWriteErrors(List<WriteStatus> statuses) {
|
||||||
|
assertAll(statuses.stream().map(status -> () ->
|
||||||
|
assertFalse(status.hasErrors(), "Errors found in write of " + status.getFileId())));
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -60,6 +60,7 @@ import java.util.Set;
|
|||||||
import java.util.function.Function;
|
import java.util.function.Function;
|
||||||
import java.util.stream.Collectors;
|
import java.util.stream.Collectors;
|
||||||
|
|
||||||
|
import static org.apache.hudi.testutils.Assertions.assertNoWriteErrors;
|
||||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||||
@@ -142,18 +143,6 @@ public class HoodieClientTestBase extends HoodieClientTestHarness {
|
|||||||
return table;
|
return table;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* Assert no failures in writing hoodie files.
|
|
||||||
*
|
|
||||||
* @param statuses List of Write Status
|
|
||||||
*/
|
|
||||||
public static void assertNoWriteErrors(List<WriteStatus> statuses) {
|
|
||||||
// Verify there are no errors
|
|
||||||
for (WriteStatus status : statuses) {
|
|
||||||
assertFalse(status.hasErrors(), "Errors found in write of " + status.getFileId());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
public void assertPartitionMetadataForRecords(List<HoodieRecord> inputRecords, FileSystem fs) throws IOException {
|
public void assertPartitionMetadataForRecords(List<HoodieRecord> inputRecords, FileSystem fs) throws IOException {
|
||||||
Set<String> partitionPathSet = inputRecords.stream()
|
Set<String> partitionPathSet = inputRecords.stream()
|
||||||
.map(HoodieRecord::getPartitionPath)
|
.map(HoodieRecord::getPartitionPath)
|
||||||
|
|||||||
Reference in New Issue
Block a user