lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-814] Migrate hudi-client tests to JUnit 5 (#1570)

Browse Source
This commit is contained in:
Raymond Xu
2020-04-28 22:57:28 -07:00
committed by GitHub
parent 06dae30297
commit 69b16309c8
4 changed files with 44 additions and 46 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
hudi-client/src/test/java/org/apache/hudi/common/config/TestHoodieWriteConfig.java
View File

@@ -22,7 +22,7 @@ import org.apache.hudi.config.HoodieCompactionConfig;
import org.apache.hudi.config.HoodieWriteConfig; import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.config.HoodieWriteConfig.Builder; import org.apache.hudi.config.HoodieWriteConfig.Builder;
import org.junit.Test; import org.junit.jupiter.api.Test;
import java.io.ByteArrayInputStream; import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream; import java.io.ByteArrayOutputStream;
@@ -32,7 +32,7 @@ import java.util.HashMap;
import java.util.Map; import java.util.Map;
import java.util.Properties; import java.util.Properties;
import static org.junit.Assert.assertEquals; import static org.junit.jupiter.api.Assertions.assertEquals;
public class TestHoodieWriteConfig { public class TestHoodieWriteConfig {
@@ -52,8 +52,8 @@ public class TestHoodieWriteConfig {
inputStream.close(); inputStream.close();
} }
HoodieWriteConfig config = builder.build(); HoodieWriteConfig config = builder.build();
assertEquals(config.getMaxCommitsToKeep(), 5); assertEquals(5, config.getMaxCommitsToKeep());
assertEquals(config.getMinCommitsToKeep(), 2); assertEquals(2, config.getMinCommitsToKeep());
} }
private ByteArrayOutputStream saveParamsIntoOutputStream(Map<String, String> params) throws IOException { private ByteArrayOutputStream saveParamsIntoOutputStream(Map<String, String> params) throws IOException {

24
hudi-client/src/test/java/org/apache/hudi/index/bloom/TestBucketizedBloomCheckPartitioner.java
View File

@@ -20,7 +20,7 @@ package org.apache.hudi.index.bloom;
import org.apache.hudi.common.util.collection.Pair; import org.apache.hudi.common.util.collection.Pair;
import org.junit.Test; import org.junit.jupiter.api.Test;
import java.util.HashMap; import java.util.HashMap;
import java.util.List; import java.util.List;
@@ -28,9 +28,9 @@ import java.util.Map;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import java.util.stream.IntStream; import java.util.stream.IntStream;
import static org.junit.Assert.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertArrayEquals;
import static org.junit.Assert.assertEquals; import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.Assert.assertTrue; import static org.junit.jupiter.api.Assertions.assertTrue;
public class TestBucketizedBloomCheckPartitioner { public class TestBucketizedBloomCheckPartitioner {
@@ -45,12 +45,12 @@ public class TestBucketizedBloomCheckPartitioner {
}; };
BucketizedBloomCheckPartitioner p = new BucketizedBloomCheckPartitioner(4, fileToComparisons, 10); BucketizedBloomCheckPartitioner p = new BucketizedBloomCheckPartitioner(4, fileToComparisons, 10);
Map<String, List<Integer>> assignments = p.getFileGroupToPartitions(); Map<String, List<Integer>> assignments = p.getFileGroupToPartitions();
assertEquals("f1 should have 4 buckets", 4, assignments.get("f1").size()); assertEquals(4, assignments.get("f1").size(), "f1 should have 4 buckets");
assertEquals("f2 should have 4 buckets", 4, assignments.get("f2").size()); assertEquals(4, assignments.get("f2").size(), "f2 should have 4 buckets");
assertEquals("f3 should have 2 buckets", 2, assignments.get("f3").size()); assertEquals(2, assignments.get("f3").size(), "f3 should have 2 buckets");
assertArrayEquals("f1 spread across 3 partitions", new Integer[] {0, 0, 1, 3}, assignments.get("f1").toArray()); assertArrayEquals(new Integer[] {0, 0, 1, 3}, assignments.get("f1").toArray(), "f1 spread across 3 partitions");
assertArrayEquals("f2 spread across 3 partitions", new Integer[] {1, 2, 2, 0}, assignments.get("f2").toArray()); assertArrayEquals(new Integer[] {1, 2, 2, 0}, assignments.get("f2").toArray(), "f2 spread across 3 partitions");
assertArrayEquals("f3 spread across 2 partitions", new Integer[] {3, 1}, assignments.get("f3").toArray()); assertArrayEquals(new Integer[] {3, 1}, assignments.get("f3").toArray(), "f3 spread across 2 partitions");
} }
@Test @Test
@@ -78,7 +78,7 @@ public class TestBucketizedBloomCheckPartitioner {
} }
}; };
BucketizedBloomCheckPartitioner p = new BucketizedBloomCheckPartitioner(10000, comparisons1, 10); BucketizedBloomCheckPartitioner p = new BucketizedBloomCheckPartitioner(10000, comparisons1, 10);
assertEquals("num partitions must equal total buckets", 100, p.numPartitions()); assertEquals(100, p.numPartitions(), "num partitions must equal total buckets");
} }
@Test @Test
@@ -92,7 +92,7 @@ public class TestBucketizedBloomCheckPartitioner {
IntStream.range(0, 100000).forEach(f -> { IntStream.range(0, 100000).forEach(f -> {
int partition = p.getPartition(Pair.of("f" + f, "value")); int partition = p.getPartition(Pair.of("f" + f, "value"));
assertTrue("partition is out of range: " + partition, 0 <= partition && partition <= 1000); assertTrue(0 <= partition && partition <= 1000, "partition is out of range: " + partition);
}); });
} }

4
hudi-client/src/test/java/org/apache/hudi/index/bloom/TestKeyRangeLookupTree.java
View File

@@ -18,7 +18,7 @@
package org.apache.hudi.index.bloom; package org.apache.hudi.index.bloom;
import org.junit.Test; import org.junit.jupiter.api.Test;
import java.util.Collections; import java.util.Collections;
import java.util.HashMap; import java.util.HashMap;
@@ -27,7 +27,7 @@ import java.util.Map;
import java.util.Random; import java.util.Random;
import java.util.UUID; import java.util.UUID;
import static junit.framework.TestCase.assertEquals; import static org.junit.jupiter.api.Assertions.assertEquals;
/** /**
* Tests {@link KeyRangeLookupTree}. * Tests {@link KeyRangeLookupTree}.

54
hudi-client/src/test/java/org/apache/hudi/table/action/compact/strategy/TestHoodieCompactionStrategy.java
View File

@@ -27,8 +27,7 @@ import org.apache.hudi.config.HoodieCompactionConfig;
import org.apache.hudi.config.HoodieWriteConfig; import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.Path;
import org.junit.Assert; import org.junit.jupiter.api.Test;
import org.junit.Test;
import java.text.SimpleDateFormat; import java.text.SimpleDateFormat;
import java.util.ArrayList; import java.util.ArrayList;
@@ -42,8 +41,8 @@ import java.util.Random;
import java.util.UUID; import java.util.UUID;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import static org.junit.Assert.assertEquals; import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.Assert.assertTrue; import static org.junit.jupiter.api.Assertions.assertTrue;
public class TestHoodieCompactionStrategy { public class TestHoodieCompactionStrategy {
@@ -63,7 +62,7 @@ public class TestHoodieCompactionStrategy {
.withCompactionConfig(HoodieCompactionConfig.newBuilder().withCompactionStrategy(strategy).build()).build(); .withCompactionConfig(HoodieCompactionConfig.newBuilder().withCompactionStrategy(strategy).build()).build();
List<HoodieCompactionOperation> operations = createCompactionOperations(writeConfig, sizesMap); List<HoodieCompactionOperation> operations = createCompactionOperations(writeConfig, sizesMap);
List<HoodieCompactionOperation> returned = strategy.orderAndFilter(writeConfig, operations, new ArrayList<>()); List<HoodieCompactionOperation> returned = strategy.orderAndFilter(writeConfig, operations, new ArrayList<>());
assertEquals("UnBounded should not re-order or filter", operations, returned); assertEquals(operations, returned, "UnBounded should not re-order or filter");
} }
@Test @Test
@@ -80,13 +79,13 @@ public class TestHoodieCompactionStrategy {
List<HoodieCompactionOperation> operations = createCompactionOperations(writeConfig, sizesMap); List<HoodieCompactionOperation> operations = createCompactionOperations(writeConfig, sizesMap);
List<HoodieCompactionOperation> returned = strategy.orderAndFilter(writeConfig, operations, new ArrayList<>()); List<HoodieCompactionOperation> returned = strategy.orderAndFilter(writeConfig, operations, new ArrayList<>());
assertTrue("BoundedIOCompaction should have resulted in fewer compactions", returned.size() < operations.size()); assertTrue(returned.size() < operations.size(), "BoundedIOCompaction should have resulted in fewer compactions");
assertEquals("BoundedIOCompaction should have resulted in 2 compactions being chosen", 2, returned.size()); assertEquals(2, returned.size(), "BoundedIOCompaction should have resulted in 2 compactions being chosen");
// Total size of all the log files // Total size of all the log files
Long returnedSize = returned.stream().map(s -> s.getMetrics().get(BoundedIOCompactionStrategy.TOTAL_IO_MB)) Long returnedSize = returned.stream().map(s -> s.getMetrics().get(BoundedIOCompactionStrategy.TOTAL_IO_MB))
.map(Double::longValue).reduce(Long::sum).orElse(0L); .map(Double::longValue).reduce(Long::sum).orElse(0L);
assertEquals("Should chose the first 2 compactions which should result in a total IO of 690 MB", 610, assertEquals(610, (long) returnedSize,
(long) returnedSize); "Should chose the first 2 compactions which should result in a total IO of 690 MB");
} }
@Test @Test
@@ -103,14 +102,14 @@ public class TestHoodieCompactionStrategy {
List<HoodieCompactionOperation> operations = createCompactionOperations(writeConfig, sizesMap); List<HoodieCompactionOperation> operations = createCompactionOperations(writeConfig, sizesMap);
List<HoodieCompactionOperation> returned = strategy.orderAndFilter(writeConfig, operations, new ArrayList<>()); List<HoodieCompactionOperation> returned = strategy.orderAndFilter(writeConfig, operations, new ArrayList<>());
assertTrue("LogFileSizeBasedCompactionStrategy should have resulted in fewer compactions", assertTrue(returned.size() < operations.size(),
returned.size() < operations.size()); "LogFileSizeBasedCompactionStrategy should have resulted in fewer compactions");
assertEquals("LogFileSizeBasedCompactionStrategy should have resulted in 1 compaction", 1, returned.size()); assertEquals(1, returned.size(), "LogFileSizeBasedCompactionStrategy should have resulted in 1 compaction");
// Total size of all the log files // Total size of all the log files
Long returnedSize = returned.stream().map(s -> s.getMetrics().get(BoundedIOCompactionStrategy.TOTAL_IO_MB)) Long returnedSize = returned.stream().map(s -> s.getMetrics().get(BoundedIOCompactionStrategy.TOTAL_IO_MB))
.map(Double::longValue).reduce(Long::sum).orElse(0L); .map(Double::longValue).reduce(Long::sum).orElse(0L);
assertEquals("Should chose the first 2 compactions which should result in a total IO of 690 MB", 1204, assertEquals(1204, (long) returnedSize,
(long) returnedSize); "Should chose the first 2 compactions which should result in a total IO of 690 MB");
} }
@Test @Test
@@ -137,14 +136,14 @@ public class TestHoodieCompactionStrategy {
List<HoodieCompactionOperation> operations = createCompactionOperations(writeConfig, sizesMap, keyToPartitionMap); List<HoodieCompactionOperation> operations = createCompactionOperations(writeConfig, sizesMap, keyToPartitionMap);
List<HoodieCompactionOperation> returned = strategy.orderAndFilter(writeConfig, operations, new ArrayList<>()); List<HoodieCompactionOperation> returned = strategy.orderAndFilter(writeConfig, operations, new ArrayList<>());
assertTrue("DayBasedCompactionStrategy should have resulted in fewer compactions", assertTrue(returned.size() < operations.size(),
returned.size() < operations.size()); "DayBasedCompactionStrategy should have resulted in fewer compactions");
Assert.assertEquals("DayBasedCompactionStrategy should have resulted in fewer compactions", returned.size(), 2); assertEquals(2, returned.size(), "DayBasedCompactionStrategy should have resulted in fewer compactions");
int comparision = strategy.getComparator().compare(returned.get(returned.size() - 1).getPartitionPath(), int comparision = strategy.getComparator().compare(returned.get(returned.size() - 1).getPartitionPath(),
returned.get(0).getPartitionPath()); returned.get(0).getPartitionPath());
// Either the partition paths are sorted in descending order or they are equal // Either the partition paths are sorted in descending order or they are equal
assertTrue("DayBasedCompactionStrategy should sort partitions in descending order", comparision >= 0); assertTrue(comparision >= 0, "DayBasedCompactionStrategy should sort partitions in descending order");
} }
@Test @Test
@@ -185,15 +184,15 @@ public class TestHoodieCompactionStrategy {
List<HoodieCompactionOperation> operations = createCompactionOperations(writeConfig, sizesMap, keyToPartitionMap); List<HoodieCompactionOperation> operations = createCompactionOperations(writeConfig, sizesMap, keyToPartitionMap);
List<HoodieCompactionOperation> returned = strategy.orderAndFilter(writeConfig, operations, new ArrayList<>()); List<HoodieCompactionOperation> returned = strategy.orderAndFilter(writeConfig, operations, new ArrayList<>());
assertTrue("BoundedPartitionAwareCompactionStrategy should have resulted in fewer compactions", assertTrue(returned.size() < operations.size(),
returned.size() < operations.size()); "BoundedPartitionAwareCompactionStrategy should have resulted in fewer compactions");
Assert.assertEquals("BoundedPartitionAwareCompactionStrategy should have resulted in fewer compactions", assertEquals(5, returned.size(),
returned.size(), 5); "BoundedPartitionAwareCompactionStrategy should have resulted in fewer compactions");
int comparision = strategy.getComparator().compare(returned.get(returned.size() - 1).getPartitionPath(), int comparision = strategy.getComparator().compare(returned.get(returned.size() - 1).getPartitionPath(),
returned.get(0).getPartitionPath()); returned.get(0).getPartitionPath());
// Either the partition paths are sorted in descending order or they are equal // Either the partition paths are sorted in descending order or they are equal
assertTrue("BoundedPartitionAwareCompactionStrategy should sort partitions in descending order", comparision >= 0); assertTrue(comparision >= 0, "BoundedPartitionAwareCompactionStrategy should sort partitions in descending order");
} }
@Test @Test
@@ -234,12 +233,11 @@ public class TestHoodieCompactionStrategy {
List<HoodieCompactionOperation> operations = createCompactionOperations(writeConfig, sizesMap, keyToPartitionMap); List<HoodieCompactionOperation> operations = createCompactionOperations(writeConfig, sizesMap, keyToPartitionMap);
List<HoodieCompactionOperation> returned = strategy.orderAndFilter(writeConfig, operations, new ArrayList<>()); List<HoodieCompactionOperation> returned = strategy.orderAndFilter(writeConfig, operations, new ArrayList<>());
assertTrue( assertTrue(returned.size() < operations.size(),
"UnBoundedPartitionAwareCompactionStrategy should not include last " "UnBoundedPartitionAwareCompactionStrategy should not include last "
+ writeConfig.getTargetPartitionsPerDayBasedCompaction() + " partitions or later partitions from today", + writeConfig.getTargetPartitionsPerDayBasedCompaction() + " partitions or later partitions from today");
returned.size() < operations.size()); assertEquals(1, returned.size(),
Assert.assertEquals("BoundedPartitionAwareCompactionStrategy should have resulted in 1 compaction", returned.size(), "BoundedPartitionAwareCompactionStrategy should have resulted in 1 compaction");
1);
} }
private List<HoodieCompactionOperation> createCompactionOperations(HoodieWriteConfig config, private List<HoodieCompactionOperation> createCompactionOperations(HoodieWriteConfig config,