lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-809] Migrate CommonTestHarness to JUnit 5 (#1530)

Browse Source
This commit is contained in:
Raymond Xu
2020-04-21 23:10:25 -07:00
committed by GitHub
parent 2a56f82908
commit 6e15eebd81
14 changed files with 459 additions and 429 deletions
Download Patch File Download Diff File Expand all files Collapse all files

54
hudi-common/src/test/java/org/apache/hudi/common/table/TestHoodieTableMetaClient.java
View File

@@ -18,41 +18,41 @@
package org.apache.hudi.common.table; package org.apache.hudi.common.table;
import org.apache.hudi.common.HoodieCommonTestHarness;
import org.apache.hudi.common.model.HoodieTestUtils; import org.apache.hudi.common.model.HoodieTestUtils;
import org.apache.hudi.common.table.timeline.HoodieActiveTimeline; import org.apache.hudi.common.table.timeline.HoodieActiveTimeline;
import org.apache.hudi.common.table.timeline.HoodieInstant; import org.apache.hudi.common.table.timeline.HoodieInstant;
import org.apache.hudi.common.table.timeline.HoodieTimeline; import org.apache.hudi.common.table.timeline.HoodieTimeline;
import org.apache.hudi.common.testutils.HoodieCommonTestHarnessJunit5;
import org.apache.hudi.common.util.Option; import org.apache.hudi.common.util.Option;
import org.junit.Before; import org.junit.jupiter.api.BeforeEach;
import org.junit.Test; import org.junit.jupiter.api.Test;
import java.io.IOException; import java.io.IOException;
import static org.junit.Assert.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertArrayEquals;
import static org.junit.Assert.assertEquals; import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.Assert.assertFalse; import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.Assert.assertNotEquals; import static org.junit.jupiter.api.Assertions.assertNotEquals;
import static org.junit.Assert.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.Assert.assertTrue; import static org.junit.jupiter.api.Assertions.assertTrue;
/** /**
* Tests hoodie table meta client {@link HoodieTableMetaClient}. * Tests hoodie table meta client {@link HoodieTableMetaClient}.
*/ */
public class TestHoodieTableMetaClient extends HoodieCommonTestHarness { public class TestHoodieTableMetaClient extends HoodieCommonTestHarnessJunit5 {
@Before @BeforeEach
public void init() throws IOException { public void init() throws IOException {
initMetaClient(); initMetaClient();
} }
@Test @Test
public void checkMetadata() { public void checkMetadata() {
assertEquals("Table name should be raw_trips", HoodieTestUtils.RAW_TRIPS_TEST_NAME, assertEquals(HoodieTestUtils.RAW_TRIPS_TEST_NAME, metaClient.getTableConfig().getTableName(), "Table name should be raw_trips");
metaClient.getTableConfig().getTableName()); assertEquals(basePath, metaClient.getBasePath(), "Basepath should be the one assigned");
assertEquals("Basepath should be the one assigned", basePath, metaClient.getBasePath()); assertEquals(basePath + "/.hoodie", metaClient.getMetaPath(), "Metapath should be ${basepath}/.hoodie");
assertEquals("Metapath should be ${basepath}/.hoodie", basePath + "/.hoodie", metaClient.getMetaPath());
} }
@Test @Test
@@ -67,16 +67,15 @@ public class TestHoodieTableMetaClient extends HoodieCommonTestHarness {
commitTimeline.saveAsComplete(instant, Option.of("test-detail".getBytes())); commitTimeline.saveAsComplete(instant, Option.of("test-detail".getBytes()));
commitTimeline = commitTimeline.reload(); commitTimeline = commitTimeline.reload();
HoodieInstant completedInstant = HoodieTimeline.getCompletedInstant(instant); HoodieInstant completedInstant = HoodieTimeline.getCompletedInstant(instant);
assertEquals("Commit should be 1 and completed", completedInstant, commitTimeline.getInstants().findFirst().get()); assertEquals(completedInstant, commitTimeline.getInstants().findFirst().get(), "Commit should be 1 and completed");
assertArrayEquals("Commit value should be \"test-detail\"", "test-detail".getBytes(), assertArrayEquals("test-detail".getBytes(), commitTimeline.getInstantDetails(completedInstant).get(), "Commit value should be \"test-detail\"");
commitTimeline.getInstantDetails(completedInstant).get());
} }
@Test @Test
public void checkCommitTimeline() { public void checkCommitTimeline() {
HoodieActiveTimeline activeTimeline = metaClient.getActiveTimeline(); HoodieActiveTimeline activeTimeline = metaClient.getActiveTimeline();
HoodieTimeline activeCommitTimeline = activeTimeline.getCommitTimeline(); HoodieTimeline activeCommitTimeline = activeTimeline.getCommitTimeline();
assertTrue("Should be empty commit timeline", activeCommitTimeline.empty()); assertTrue(activeCommitTimeline.empty(), "Should be empty commit timeline");
HoodieInstant instant = new HoodieInstant(true, HoodieTimeline.COMMIT_ACTION, "1"); HoodieInstant instant = new HoodieInstant(true, HoodieTimeline.COMMIT_ACTION, "1");
activeTimeline.createNewInstant(instant); activeTimeline.createNewInstant(instant);
@@ -85,21 +84,20 @@ public class TestHoodieTableMetaClient extends HoodieCommonTestHarness {
// Commit timeline should not auto-reload every time getActiveCommitTimeline(), it should be cached // Commit timeline should not auto-reload every time getActiveCommitTimeline(), it should be cached
activeTimeline = metaClient.getActiveTimeline(); activeTimeline = metaClient.getActiveTimeline();
activeCommitTimeline = activeTimeline.getCommitTimeline(); activeCommitTimeline = activeTimeline.getCommitTimeline();
assertTrue("Should be empty commit timeline", activeCommitTimeline.empty()); assertTrue(activeCommitTimeline.empty(), "Should be empty commit timeline");
HoodieInstant completedInstant = HoodieTimeline.getCompletedInstant(instant); HoodieInstant completedInstant = HoodieTimeline.getCompletedInstant(instant);
activeTimeline = activeTimeline.reload(); activeTimeline = activeTimeline.reload();
activeCommitTimeline = activeTimeline.getCommitTimeline(); activeCommitTimeline = activeTimeline.getCommitTimeline();
assertFalse("Should be the 1 commit we made", activeCommitTimeline.empty()); assertFalse(activeCommitTimeline.empty(), "Should be the 1 commit we made");
assertEquals("Commit should be 1", completedInstant, activeCommitTimeline.getInstants().findFirst().get()); assertEquals(completedInstant, activeCommitTimeline.getInstants().findFirst().get(), "Commit should be 1");
assertArrayEquals("Commit value should be \"test-detail\"", "test-detail".getBytes(), assertArrayEquals("test-detail".getBytes(), activeCommitTimeline.getInstantDetails(completedInstant).get(), "Commit value should be \"test-detail\"");
activeCommitTimeline.getInstantDetails(completedInstant).get());
} }
@Test @Test
public void testEquals() throws IOException { public void testEquals() throws IOException {
HoodieTableMetaClient metaClient1 = HoodieTestUtils.init(folder.getRoot().getAbsolutePath(), getTableType()); HoodieTableMetaClient metaClient1 = HoodieTestUtils.init(tempDir.toAbsolutePath().toString(), getTableType());
HoodieTableMetaClient metaClient2 = HoodieTestUtils.init(folder.getRoot().getAbsolutePath(), getTableType()); HoodieTableMetaClient metaClient2 = HoodieTestUtils.init(tempDir.toAbsolutePath().toString(), getTableType());
assertEquals(metaClient1, metaClient1); assertEquals(metaClient1, metaClient1);
assertEquals(metaClient1, metaClient2); assertEquals(metaClient1, metaClient2);
assertNotEquals(metaClient1, null); assertNotEquals(metaClient1, null);
@@ -108,8 +106,8 @@ public class TestHoodieTableMetaClient extends HoodieCommonTestHarness {
@Test @Test
public void testToString() throws IOException { public void testToString() throws IOException {
HoodieTableMetaClient metaClient1 = HoodieTestUtils.init(folder.getRoot().getAbsolutePath(), getTableType()); HoodieTableMetaClient metaClient1 = HoodieTestUtils.init(tempDir.toAbsolutePath().toString(), getTableType());
HoodieTableMetaClient metaClient2 = HoodieTestUtils.init(folder.getRoot().getAbsolutePath(), getTableType()); HoodieTableMetaClient metaClient2 = HoodieTestUtils.init(tempDir.toAbsolutePath().toString(), getTableType());
assertEquals(metaClient1.toString(), metaClient2.toString()); assertEquals(metaClient1.toString(), metaClient2.toString());
assertNotEquals(metaClient1.toString(), new Object().toString()); assertNotEquals(metaClient1.toString(), new Object().toString());
} }

24
hudi-common/src/test/java/org/apache/hudi/common/table/TestTimelineLayout.java
View File

@@ -24,13 +24,15 @@ import org.apache.hudi.common.table.timeline.HoodieTimeline;
import org.apache.hudi.common.table.timeline.TimelineLayout; import org.apache.hudi.common.table.timeline.TimelineLayout;
import org.apache.hudi.common.table.timeline.versioning.TimelineLayoutVersion; import org.apache.hudi.common.table.timeline.versioning.TimelineLayoutVersion;
import org.junit.Assert; import org.junit.jupiter.api.Test;
import org.junit.Test;
import java.util.Arrays; import java.util.Arrays;
import java.util.List; import java.util.List;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
public class TestTimelineLayout { public class TestTimelineLayout {
@Test @Test
@@ -57,23 +59,23 @@ public class TestTimelineLayout {
List<HoodieInstant> layout0Instants = TimelineLayout.getLayout(new TimelineLayoutVersion(0)) List<HoodieInstant> layout0Instants = TimelineLayout.getLayout(new TimelineLayoutVersion(0))
.filterHoodieInstants(rawInstants.stream()).collect(Collectors.toList()); .filterHoodieInstants(rawInstants.stream()).collect(Collectors.toList());
Assert.assertEquals(rawInstants, layout0Instants); assertEquals(rawInstants, layout0Instants);
List<HoodieInstant> layout1Instants = TimelineLayout.getLayout(TimelineLayoutVersion.CURR_LAYOUT_VERSION) List<HoodieInstant> layout1Instants = TimelineLayout.getLayout(TimelineLayoutVersion.CURR_LAYOUT_VERSION)
.filterHoodieInstants(rawInstants.stream()).collect(Collectors.toList()); .filterHoodieInstants(rawInstants.stream()).collect(Collectors.toList());
Assert.assertEquals(7, layout1Instants.size()); assertEquals(7, layout1Instants.size());
Assert.assertTrue(layout1Instants.contains( assertTrue(layout1Instants.contains(
new HoodieInstant(State.INFLIGHT, HoodieTimeline.DELTA_COMMIT_ACTION, "007"))); new HoodieInstant(State.INFLIGHT, HoodieTimeline.DELTA_COMMIT_ACTION, "007")));
Assert.assertTrue(layout1Instants.contains( assertTrue(layout1Instants.contains(
new HoodieInstant(State.INFLIGHT, HoodieTimeline.COMPACTION_ACTION, "006"))); new HoodieInstant(State.INFLIGHT, HoodieTimeline.COMPACTION_ACTION, "006")));
Assert.assertTrue(layout1Instants.contains( assertTrue(layout1Instants.contains(
new HoodieInstant(State.COMPLETED, HoodieTimeline.DELTA_COMMIT_ACTION, "005"))); new HoodieInstant(State.COMPLETED, HoodieTimeline.DELTA_COMMIT_ACTION, "005")));
Assert.assertTrue(layout1Instants.contains( assertTrue(layout1Instants.contains(
new HoodieInstant(State.INFLIGHT, HoodieTimeline.CLEAN_ACTION, "004"))); new HoodieInstant(State.INFLIGHT, HoodieTimeline.CLEAN_ACTION, "004")));
Assert.assertTrue(layout1Instants.contains( assertTrue(layout1Instants.contains(
new HoodieInstant(State.COMPLETED, HoodieTimeline.COMMIT_ACTION, "003"))); new HoodieInstant(State.COMPLETED, HoodieTimeline.COMMIT_ACTION, "003")));
Assert.assertTrue(layout1Instants.contains( assertTrue(layout1Instants.contains(
new HoodieInstant(State.COMPLETED, HoodieTimeline.DELTA_COMMIT_ACTION, "002"))); new HoodieInstant(State.COMPLETED, HoodieTimeline.DELTA_COMMIT_ACTION, "002")));
Assert.assertTrue(layout1Instants.contains( assertTrue(layout1Instants.contains(
new HoodieInstant(State.COMPLETED, HoodieTimeline.CLEAN_ACTION, "001"))); new HoodieInstant(State.COMPLETED, HoodieTimeline.CLEAN_ACTION, "001")));
} }
} }

333
hudi-common/src/test/java/org/apache/hudi/common/table/view/TestHoodieTableFileSystemView.java
View File

@@ -19,7 +19,6 @@
package org.apache.hudi.common.table.view; package org.apache.hudi.common.table.view;
import org.apache.hudi.avro.model.HoodieCompactionPlan; import org.apache.hudi.avro.model.HoodieCompactionPlan;
import org.apache.hudi.common.HoodieCommonTestHarness;
import org.apache.hudi.common.fs.FSUtils; import org.apache.hudi.common.fs.FSUtils;
import org.apache.hudi.common.model.CompactionOperation; import org.apache.hudi.common.model.CompactionOperation;
import org.apache.hudi.common.model.FileSlice; import org.apache.hudi.common.model.FileSlice;
@@ -35,6 +34,7 @@ import org.apache.hudi.common.table.timeline.HoodieTimeline;
import org.apache.hudi.common.table.timeline.TimelineMetadataUtils; import org.apache.hudi.common.table.timeline.TimelineMetadataUtils;
import org.apache.hudi.common.table.view.TableFileSystemView.BaseFileOnlyView; import org.apache.hudi.common.table.view.TableFileSystemView.BaseFileOnlyView;
import org.apache.hudi.common.table.view.TableFileSystemView.SliceView; import org.apache.hudi.common.table.view.TableFileSystemView.SliceView;
import org.apache.hudi.common.testutils.HoodieCommonTestHarnessJunit5;
import org.apache.hudi.common.util.CompactionUtils; import org.apache.hudi.common.util.CompactionUtils;
import org.apache.hudi.common.util.Option; import org.apache.hudi.common.util.Option;
import org.apache.hudi.common.util.collection.Pair; import org.apache.hudi.common.util.collection.Pair;
@@ -43,9 +43,8 @@ import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.Path;
import org.apache.log4j.LogManager; import org.apache.log4j.LogManager;
import org.apache.log4j.Logger; import org.apache.log4j.Logger;
import org.junit.Assert; import org.junit.jupiter.api.BeforeEach;
import org.junit.Before; import org.junit.jupiter.api.Test;
import org.junit.Test;
import java.io.File; import java.io.File;
import java.io.IOException; import java.io.IOException;
@@ -59,15 +58,15 @@ import java.util.UUID;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import java.util.stream.Stream; import java.util.stream.Stream;
import static org.junit.Assert.assertEquals; import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.Assert.assertFalse; import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.Assert.assertTrue; import static org.junit.jupiter.api.Assertions.assertTrue;
/** /**
* Tests hoodie table file system view {@link HoodieTableFileSystemView}. * Tests hoodie table file system view {@link HoodieTableFileSystemView}.
*/ */
@SuppressWarnings("ResultOfMethodCallIgnored") @SuppressWarnings("ResultOfMethodCallIgnored")
public class TestHoodieTableFileSystemView extends HoodieCommonTestHarness { public class TestHoodieTableFileSystemView extends HoodieCommonTestHarnessJunit5 {
private static final Logger LOG = LogManager.getLogger(TestHoodieTableFileSystemView.class); private static final Logger LOG = LogManager.getLogger(TestHoodieTableFileSystemView.class);
@@ -77,7 +76,7 @@ public class TestHoodieTableFileSystemView extends HoodieCommonTestHarness {
protected BaseFileOnlyView roView; protected BaseFileOnlyView roView;
protected SliceView rtView; protected SliceView rtView;
@Before @BeforeEach
public void init() throws IOException { public void init() throws IOException {
initMetaClient(); initMetaClient();
refreshFsView(); refreshFsView();
@@ -134,49 +133,49 @@ public class TestHoodieTableFileSystemView extends HoodieCommonTestHarness {
refreshFsView(); refreshFsView();
List<HoodieBaseFile> dataFiles = roView.getLatestBaseFiles().collect(Collectors.toList()); List<HoodieBaseFile> dataFiles = roView.getLatestBaseFiles().collect(Collectors.toList());
assertTrue("No data file expected", dataFiles.isEmpty()); assertTrue(dataFiles.isEmpty(), "No data file expected");
List<FileSlice> fileSliceList = rtView.getLatestFileSlices(partitionPath).collect(Collectors.toList()); List<FileSlice> fileSliceList = rtView.getLatestFileSlices(partitionPath).collect(Collectors.toList());
assertEquals(1, fileSliceList.size()); assertEquals(1, fileSliceList.size());
FileSlice fileSlice = fileSliceList.get(0); FileSlice fileSlice = fileSliceList.get(0);
assertEquals("File-Id must be set correctly", fileId, fileSlice.getFileId()); assertEquals(fileId, fileSlice.getFileId(), "File-Id must be set correctly");
assertFalse("Data file for base instant must be present", fileSlice.getBaseFile().isPresent()); assertFalse(fileSlice.getBaseFile().isPresent(), "Data file for base instant must be present");
assertEquals("Base Instant for file-group set correctly", instantTime1, fileSlice.getBaseInstantTime()); assertEquals(instantTime1, fileSlice.getBaseInstantTime(), "Base Instant for file-group set correctly");
List<HoodieLogFile> logFiles = fileSlice.getLogFiles().collect(Collectors.toList()); List<HoodieLogFile> logFiles = fileSlice.getLogFiles().collect(Collectors.toList());
assertEquals("Correct number of log-files shows up in file-slice", 2, logFiles.size()); assertEquals(2, logFiles.size(), "Correct number of log-files shows up in file-slice");
assertEquals("Log File Order check", fileName2, logFiles.get(0).getFileName()); assertEquals(fileName2, logFiles.get(0).getFileName(), "Log File Order check");
assertEquals("Log File Order check", fileName1, logFiles.get(1).getFileName()); assertEquals(fileName1, logFiles.get(1).getFileName(), "Log File Order check");
// Check Merged File Slices API // Check Merged File Slices API
fileSliceList = fileSliceList =
rtView.getLatestMergedFileSlicesBeforeOrOn(partitionPath, deltaInstantTime2).collect(Collectors.toList()); rtView.getLatestMergedFileSlicesBeforeOrOn(partitionPath, deltaInstantTime2).collect(Collectors.toList());
assertEquals(1, fileSliceList.size()); assertEquals(1, fileSliceList.size());
fileSlice = fileSliceList.get(0); fileSlice = fileSliceList.get(0);
assertEquals("File-Id must be set correctly", fileId, fileSlice.getFileId()); assertEquals(fileId, fileSlice.getFileId(), "File-Id must be set correctly");
assertFalse("Data file for base instant must be present", fileSlice.getBaseFile().isPresent()); assertFalse(fileSlice.getBaseFile().isPresent(), "Data file for base instant must be present");
assertEquals("Base Instant for file-group set correctly", instantTime1, fileSlice.getBaseInstantTime()); assertEquals(instantTime1, fileSlice.getBaseInstantTime(), "Base Instant for file-group set correctly");
logFiles = fileSlice.getLogFiles().collect(Collectors.toList()); logFiles = fileSlice.getLogFiles().collect(Collectors.toList());
assertEquals("Correct number of log-files shows up in file-slice", 2, logFiles.size()); assertEquals(2, logFiles.size(), "Correct number of log-files shows up in file-slice");
assertEquals("Log File Order check", fileName2, logFiles.get(0).getFileName()); assertEquals(fileName2, logFiles.get(0).getFileName(), "Log File Order check");
assertEquals("Log File Order check", fileName1, logFiles.get(1).getFileName()); assertEquals(fileName1, logFiles.get(1).getFileName(), "Log File Order check");
// Check UnCompacted File Slices API // Check UnCompacted File Slices API
fileSliceList = rtView.getLatestUnCompactedFileSlices(partitionPath).collect(Collectors.toList()); fileSliceList = rtView.getLatestUnCompactedFileSlices(partitionPath).collect(Collectors.toList());
assertEquals(1, fileSliceList.size()); assertEquals(1, fileSliceList.size());
fileSlice = fileSliceList.get(0); fileSlice = fileSliceList.get(0);
assertEquals("File-Id must be set correctly", fileId, fileSlice.getFileId()); assertEquals(fileId, fileSlice.getFileId(), "File-Id must be set correctly");
assertFalse("Data file for base instant must be present", fileSlice.getBaseFile().isPresent()); assertFalse(fileSlice.getBaseFile().isPresent(), "Data file for base instant must be present");
assertEquals("Base Instant for file-group set correctly", instantTime1, fileSlice.getBaseInstantTime()); assertEquals(instantTime1, fileSlice.getBaseInstantTime(), "Base Instant for file-group set correctly");
logFiles = fileSlice.getLogFiles().collect(Collectors.toList()); logFiles = fileSlice.getLogFiles().collect(Collectors.toList());
assertEquals("Correct number of log-files shows up in file-slice", 2, logFiles.size()); assertEquals(2, logFiles.size(), "Correct number of log-files shows up in file-slice");
assertEquals("Log File Order check", fileName2, logFiles.get(0).getFileName()); assertEquals(fileName2, logFiles.get(0).getFileName(), "Log File Order check");
assertEquals("Log File Order check", fileName1, logFiles.get(1).getFileName()); assertEquals(fileName1, logFiles.get(1).getFileName(), "Log File Order check");
assertEquals("Total number of file-slices in view matches expected", expNumTotalFileSlices, assertEquals(expNumTotalFileSlices, rtView.getAllFileSlices(partitionPath).count(),
rtView.getAllFileSlices(partitionPath).count()); "Total number of file-slices in view matches expected");
assertEquals("Total number of data-files in view matches expected", expNumTotalDataFiles, assertEquals(expNumTotalDataFiles, roView.getAllBaseFiles(partitionPath).count(),
roView.getAllBaseFiles(partitionPath).count()); "Total number of data-files in view matches expected");
assertEquals("Total number of file-groups in view matches expected", 1, assertEquals(1, fsView.getAllFileGroups(partitionPath).count(),
fsView.getAllFileGroups(partitionPath).count()); "Total number of file-groups in view matches expected");
} }
@Test @Test
@@ -288,11 +287,10 @@ public class TestHoodieTableFileSystemView extends HoodieCommonTestHarness {
// View immediately after scheduling compaction // View immediately after scheduling compaction
refreshFsView(); refreshFsView();
List<FileSlice> slices = rtView.getLatestFileSlices(partitionPath).collect(Collectors.toList()); List<FileSlice> slices = rtView.getLatestFileSlices(partitionPath).collect(Collectors.toList());
assertEquals("Expected latest file-slices", 1, slices.size()); assertEquals(1, slices.size(), "Expected latest file-slices");
assertEquals("Base-Instant must be compaction Instant", compactionRequestedTime, assertEquals(compactionRequestedTime, slices.get(0).getBaseInstantTime(), "Base-Instant must be compaction Instant");
slices.get(0).getBaseInstantTime()); assertFalse(slices.get(0).getBaseFile().isPresent(), "Latest File Slice must not have data-file");
assertFalse("Latest File Slice must not have data-file", slices.get(0).getBaseFile().isPresent()); assertEquals(0, slices.get(0).getLogFiles().count(), "Latest File Slice must not have any log-files");
assertEquals("Latest File Slice must not have any log-files", 0, slices.get(0).getLogFiles().count());
// Fake delta-ingestion after compaction-requested // Fake delta-ingestion after compaction-requested
String deltaInstantTime4 = "5"; String deltaInstantTime4 = "5";
@@ -313,73 +311,71 @@ public class TestHoodieTableFileSystemView extends HoodieCommonTestHarness {
List<HoodieBaseFile> dataFiles = roView.getAllBaseFiles(partitionPath).collect(Collectors.toList()); List<HoodieBaseFile> dataFiles = roView.getAllBaseFiles(partitionPath).collect(Collectors.toList());
if (skipCreatingDataFile) { if (skipCreatingDataFile) {
assertTrue("No data file expected", dataFiles.isEmpty()); assertTrue(dataFiles.isEmpty(), "No data file expected");
} else { } else {
assertEquals("One data-file is expected as there is only one file-group", 1, dataFiles.size()); assertEquals(1, dataFiles.size(), "One data-file is expected as there is only one file-group");
assertEquals("Expect only valid data-file", dataFileName, dataFiles.get(0).getFileName()); assertEquals(dataFileName, dataFiles.get(0).getFileName(), "Expect only valid data-file");
} }
// Merge API Tests // Merge API Tests
List<FileSlice> fileSliceList = List<FileSlice> fileSliceList =
rtView.getLatestMergedFileSlicesBeforeOrOn(partitionPath, deltaInstantTime5).collect(Collectors.toList()); rtView.getLatestMergedFileSlicesBeforeOrOn(partitionPath, deltaInstantTime5).collect(Collectors.toList());
assertEquals("Expect file-slice to be merged", 1, fileSliceList.size()); assertEquals(1, fileSliceList.size(), "Expect file-slice to be merged");
FileSlice fileSlice = fileSliceList.get(0); FileSlice fileSlice = fileSliceList.get(0);
assertEquals(fileId, fileSlice.getFileId()); assertEquals(fileId, fileSlice.getFileId());
if (!skipCreatingDataFile) { if (!skipCreatingDataFile) {
assertEquals("Data file must be present", dataFileName, fileSlice.getBaseFile().get().getFileName()); assertEquals(dataFileName, fileSlice.getBaseFile().get().getFileName(), "Data file must be present");
} else { } else {
assertFalse("No data-file expected as it was not created", fileSlice.getBaseFile().isPresent()); assertFalse(fileSlice.getBaseFile().isPresent(), "No data-file expected as it was not created");
} }
assertEquals("Base Instant of penultimate file-slice must be base instant", instantTime1, assertEquals(instantTime1, fileSlice.getBaseInstantTime(), "Base Instant of penultimate file-slice must be base instant");
fileSlice.getBaseInstantTime());
List<HoodieLogFile> logFiles = fileSlice.getLogFiles().collect(Collectors.toList()); List<HoodieLogFile> logFiles = fileSlice.getLogFiles().collect(Collectors.toList());
assertEquals("Log files must include those after compaction request", 4, logFiles.size()); assertEquals(4, logFiles.size(), "Log files must include those after compaction request");
assertEquals("Log File Order check", fileName4, logFiles.get(0).getFileName()); assertEquals(fileName4, logFiles.get(0).getFileName(), "Log File Order check");
assertEquals("Log File Order check", fileName3, logFiles.get(1).getFileName()); assertEquals(fileName3, logFiles.get(1).getFileName(), "Log File Order check");
assertEquals("Log File Order check", fileName2, logFiles.get(2).getFileName()); assertEquals(fileName2, logFiles.get(2).getFileName(), "Log File Order check");
assertEquals("Log File Order check", fileName1, logFiles.get(3).getFileName()); assertEquals(fileName1, logFiles.get(3).getFileName(), "Log File Order check");
fileSliceList = fileSliceList =
rtView.getLatestFileSlicesBeforeOrOn(partitionPath, deltaInstantTime5, true).collect(Collectors.toList()); rtView.getLatestFileSlicesBeforeOrOn(partitionPath, deltaInstantTime5, true).collect(Collectors.toList());
assertEquals("Expect only one file-id", 1, fileSliceList.size()); assertEquals(1, fileSliceList.size(), "Expect only one file-id");
fileSlice = fileSliceList.get(0); fileSlice = fileSliceList.get(0);
assertEquals(fileId, fileSlice.getFileId()); assertEquals(fileId, fileSlice.getFileId());
assertFalse("No data-file expected in latest file-slice", fileSlice.getBaseFile().isPresent()); assertFalse(fileSlice.getBaseFile().isPresent(), "No data-file expected in latest file-slice");
assertEquals("Compaction requested instant must be base instant", compactionRequestedTime, assertEquals(compactionRequestedTime, fileSlice.getBaseInstantTime(), "Compaction requested instant must be base instant");
fileSlice.getBaseInstantTime());
logFiles = fileSlice.getLogFiles().collect(Collectors.toList()); logFiles = fileSlice.getLogFiles().collect(Collectors.toList());
assertEquals("Log files must include only those after compaction request", 2, logFiles.size()); assertEquals(2, logFiles.size(), "Log files must include only those after compaction request");
assertEquals("Log File Order check", fileName4, logFiles.get(0).getFileName()); assertEquals(fileName4, logFiles.get(0).getFileName(), "Log File Order check");
assertEquals("Log File Order check", fileName3, logFiles.get(1).getFileName()); assertEquals(fileName3, logFiles.get(1).getFileName(), "Log File Order check");
// Data Files API tests // Data Files API tests
dataFiles = roView.getLatestBaseFiles().collect(Collectors.toList()); dataFiles = roView.getLatestBaseFiles().collect(Collectors.toList());
if (skipCreatingDataFile) { if (skipCreatingDataFile) {
assertEquals("Expect no data file to be returned", 0, dataFiles.size()); assertEquals(0, dataFiles.size(), "Expect no data file to be returned");
} else { } else {
assertEquals("Expect only one data-file to be sent", 1, dataFiles.size()); assertEquals(1, dataFiles.size(), "Expect only one data-file to be sent");
dataFiles.forEach(df -> assertEquals("Expect data-file for instant 1 be returned", df.getCommitTime(), instantTime1)); dataFiles.forEach(df -> assertEquals(df.getCommitTime(), instantTime1, "Expect data-file for instant 1 be returned"));
} }
dataFiles = roView.getLatestBaseFiles(partitionPath).collect(Collectors.toList()); dataFiles = roView.getLatestBaseFiles(partitionPath).collect(Collectors.toList());
if (skipCreatingDataFile) { if (skipCreatingDataFile) {
assertEquals("Expect no data file to be returned", 0, dataFiles.size()); assertEquals(0, dataFiles.size(), "Expect no data file to be returned");
} else { } else {
assertEquals("Expect only one data-file to be sent", 1, dataFiles.size()); assertEquals(1, dataFiles.size(), "Expect only one data-file to be sent");
dataFiles.forEach(df -> assertEquals("Expect data-file for instant 1 be returned", df.getCommitTime(), instantTime1)); dataFiles.forEach(df -> assertEquals(df.getCommitTime(), instantTime1, "Expect data-file for instant 1 be returned"));
} }
dataFiles = roView.getLatestBaseFilesBeforeOrOn(partitionPath, deltaInstantTime5).collect(Collectors.toList()); dataFiles = roView.getLatestBaseFilesBeforeOrOn(partitionPath, deltaInstantTime5).collect(Collectors.toList());
if (skipCreatingDataFile) { if (skipCreatingDataFile) {
assertEquals("Expect no data file to be returned", 0, dataFiles.size()); assertEquals(0, dataFiles.size(), "Expect no data file to be returned");
} else { } else {
assertEquals("Expect only one data-file to be sent", 1, dataFiles.size()); assertEquals(1, dataFiles.size(), "Expect only one data-file to be sent");
dataFiles.forEach(df -> assertEquals("Expect data-file for instant 1 be returned", df.getCommitTime(), instantTime1)); dataFiles.forEach(df -> assertEquals(df.getCommitTime(), instantTime1, "Expect data-file for instant 1 be returned"));
} }
dataFiles = roView.getLatestBaseFilesInRange(allInstantTimes).collect(Collectors.toList()); dataFiles = roView.getLatestBaseFilesInRange(allInstantTimes).collect(Collectors.toList());
if (skipCreatingDataFile) { if (skipCreatingDataFile) {
assertEquals("Expect no data file to be returned", 0, dataFiles.size()); assertEquals(0, dataFiles.size(), "Expect no data file to be returned");
} else { } else {
assertEquals("Expect only one data-file to be sent", 1, dataFiles.size()); assertEquals(1, dataFiles.size(), "Expect only one data-file to be sent");
dataFiles.forEach(df -> assertEquals("Expect data-file for instant 1 be returned", df.getCommitTime(), instantTime1)); dataFiles.forEach(df -> assertEquals(df.getCommitTime(), instantTime1, "Expect data-file for instant 1 be returned"));
} }
// Inflight/Orphan File-groups needs to be in the view // Inflight/Orphan File-groups needs to be in the view
@@ -420,22 +416,22 @@ public class TestHoodieTableFileSystemView extends HoodieCommonTestHarness {
}).collect(Collectors.toList()); }).collect(Collectors.toList());
if (includeInvalidAndInflight) { if (includeInvalidAndInflight) {
assertEquals("Inflight/Orphan data-file is also expected", assertEquals(2 + (isCompactionInFlight ? 1 : 0) + (skipCreatingDataFile ? 0 : 1), dataFiles.size(),
2 + (isCompactionInFlight ? 1 : 0) + (skipCreatingDataFile ? 0 : 1), dataFiles.size()); "Inflight/Orphan data-file is also expected");
Set<String> fileNames = dataFiles.stream().map(HoodieBaseFile::getFileName).collect(Collectors.toSet()); Set<String> fileNames = dataFiles.stream().map(HoodieBaseFile::getFileName).collect(Collectors.toSet());
assertTrue("Expect orphan data-file to be present", fileNames.contains(orphanDataFileName)); assertTrue(fileNames.contains(orphanDataFileName), "Expect orphan data-file to be present");
assertTrue("Expect inflight data-file to be present", fileNames.contains(inflightDataFileName)); assertTrue(fileNames.contains(inflightDataFileName), "Expect inflight data-file to be present");
if (!skipCreatingDataFile) { if (!skipCreatingDataFile) {
assertTrue("Expect old committed data-file", fileNames.contains(dataFileName)); assertTrue(fileNames.contains(dataFileName), "Expect old committed data-file");
} }
if (isCompactionInFlight) { if (isCompactionInFlight) {
assertTrue("Expect inflight compacted data file to be present", fileNames.contains(compactDataFileName)); assertTrue(fileNames.contains(compactDataFileName), "Expect inflight compacted data file to be present");
} }
fileSliceList = getLatestRawFileSlices(partitionPath).collect(Collectors.toList()); fileSliceList = getLatestRawFileSlices(partitionPath).collect(Collectors.toList());
assertEquals("Expect both inflight and orphan file-slice to be included", includeInvalidAndInflight ? 5 : 1, assertEquals(includeInvalidAndInflight ? 5 : 1, fileSliceList.size(),
fileSliceList.size()); "Expect both inflight and orphan file-slice to be included");
Map<String, FileSlice> fileSliceMap = Map<String, FileSlice> fileSliceMap =
fileSliceList.stream().collect(Collectors.toMap(FileSlice::getFileId, r -> r)); fileSliceList.stream().collect(Collectors.toMap(FileSlice::getFileId, r -> r));
FileSlice orphanFileSliceWithDataFile = fileSliceMap.get(orphanFileId1); FileSlice orphanFileSliceWithDataFile = fileSliceMap.get(orphanFileId1);
@@ -443,33 +439,33 @@ public class TestHoodieTableFileSystemView extends HoodieCommonTestHarness {
FileSlice inflightFileSliceWithDataFile = fileSliceMap.get(inflightFileId1); FileSlice inflightFileSliceWithDataFile = fileSliceMap.get(inflightFileId1);
FileSlice inflightFileSliceWithLogFile = fileSliceMap.get(inflightFileId2); FileSlice inflightFileSliceWithLogFile = fileSliceMap.get(inflightFileId2);
assertEquals("Orphan File Slice with data-file check base-commit", invalidInstantId, assertEquals(invalidInstantId, orphanFileSliceWithDataFile.getBaseInstantTime(),
orphanFileSliceWithDataFile.getBaseInstantTime()); "Orphan File Slice with data-file check base-commit");
assertEquals("Orphan File Slice with data-file check data-file", orphanDataFileName, assertEquals(orphanDataFileName, orphanFileSliceWithDataFile.getBaseFile().get().getFileName(),
orphanFileSliceWithDataFile.getBaseFile().get().getFileName()); "Orphan File Slice with data-file check data-file");
assertEquals("Orphan File Slice with data-file check data-file", 0, assertEquals(0, orphanFileSliceWithDataFile.getLogFiles().count(),
orphanFileSliceWithDataFile.getLogFiles().count()); "Orphan File Slice with data-file check data-file");
assertEquals("Inflight File Slice with data-file check base-commit", inflightDeltaInstantTime, assertEquals(inflightDeltaInstantTime, inflightFileSliceWithDataFile.getBaseInstantTime(),
inflightFileSliceWithDataFile.getBaseInstantTime()); "Inflight File Slice with data-file check base-commit");
assertEquals("Inflight File Slice with data-file check data-file", inflightDataFileName, assertEquals(inflightDataFileName, inflightFileSliceWithDataFile.getBaseFile().get().getFileName(),
inflightFileSliceWithDataFile.getBaseFile().get().getFileName()); "Inflight File Slice with data-file check data-file");
assertEquals("Inflight File Slice with data-file check data-file", 0, assertEquals(0, inflightFileSliceWithDataFile.getLogFiles().count(),
inflightFileSliceWithDataFile.getLogFiles().count()); "Inflight File Slice with data-file check data-file");
assertEquals("Orphan File Slice with log-file check base-commit", invalidInstantId, assertEquals(invalidInstantId, orphanFileSliceWithLogFile.getBaseInstantTime(),
orphanFileSliceWithLogFile.getBaseInstantTime()); "Orphan File Slice with log-file check base-commit");
assertFalse("Orphan File Slice with log-file check data-file", assertFalse(orphanFileSliceWithLogFile.getBaseFile().isPresent(),
orphanFileSliceWithLogFile.getBaseFile().isPresent()); "Orphan File Slice with log-file check data-file");
logFiles = orphanFileSliceWithLogFile.getLogFiles().collect(Collectors.toList()); logFiles = orphanFileSliceWithLogFile.getLogFiles().collect(Collectors.toList());
assertEquals("Orphan File Slice with log-file check data-file", 1, logFiles.size()); assertEquals(1, logFiles.size(), "Orphan File Slice with log-file check data-file");
assertEquals("Orphan File Slice with log-file check data-file", orphanLogFileName, logFiles.get(0).getFileName()); assertEquals(orphanLogFileName, logFiles.get(0).getFileName(), "Orphan File Slice with log-file check data-file");
assertEquals("Inflight File Slice with log-file check base-commit", inflightDeltaInstantTime, assertEquals(inflightDeltaInstantTime, inflightFileSliceWithLogFile.getBaseInstantTime(),
inflightFileSliceWithLogFile.getBaseInstantTime()); "Inflight File Slice with log-file check base-commit");
assertFalse("Inflight File Slice with log-file check data-file", assertFalse(inflightFileSliceWithLogFile.getBaseFile().isPresent(),
inflightFileSliceWithLogFile.getBaseFile().isPresent()); "Inflight File Slice with log-file check data-file");
logFiles = inflightFileSliceWithLogFile.getLogFiles().collect(Collectors.toList()); logFiles = inflightFileSliceWithLogFile.getLogFiles().collect(Collectors.toList());
assertEquals("Inflight File Slice with log-file check data-file", 1, logFiles.size()); assertEquals(1, logFiles.size(), "Inflight File Slice with log-file check data-file");
assertEquals("Inflight File Slice with log-file check data-file", inflightLogFileName, assertEquals(inflightLogFileName, logFiles.get(0).getFileName(),
logFiles.get(0).getFileName()); "Inflight File Slice with log-file check data-file");
} }
compactionInstant = new HoodieInstant(State.INFLIGHT, HoodieTimeline.COMPACTION_ACTION, compactionRequestedTime); compactionInstant = new HoodieInstant(State.INFLIGHT, HoodieTimeline.COMPACTION_ACTION, compactionRequestedTime);
@@ -490,41 +486,41 @@ public class TestHoodieTableFileSystemView extends HoodieCommonTestHarness {
LOG.info("FILESLICE LIST=" + fileSliceList); LOG.info("FILESLICE LIST=" + fileSliceList);
dataFiles = fileSliceList.stream().map(FileSlice::getBaseFile).filter(Option::isPresent).map(Option::get) dataFiles = fileSliceList.stream().map(FileSlice::getBaseFile).filter(Option::isPresent).map(Option::get)
.collect(Collectors.toList()); .collect(Collectors.toList());
assertEquals("Expect only one data-files in latest view as there is only one file-group", 1, dataFiles.size()); assertEquals(1, dataFiles.size(), "Expect only one data-files in latest view as there is only one file-group");
assertEquals("Data Filename must match", compactDataFileName, dataFiles.get(0).getFileName()); assertEquals(compactDataFileName, dataFiles.get(0).getFileName(), "Data Filename must match");
assertEquals("Only one latest file-slice in the partition", 1, fileSliceList.size()); assertEquals(1, fileSliceList.size(), "Only one latest file-slice in the partition");
fileSlice = fileSliceList.get(0); fileSlice = fileSliceList.get(0);
assertEquals("Check file-Id is set correctly", fileId, fileSlice.getFileId()); assertEquals(fileId, fileSlice.getFileId(), "Check file-Id is set correctly");
assertEquals("Check data-filename is set correctly", compactDataFileName, assertEquals(compactDataFileName, fileSlice.getBaseFile().get().getFileName(),
fileSlice.getBaseFile().get().getFileName()); "Check data-filename is set correctly");
assertEquals("Ensure base-instant is now compaction request instant", compactionRequestedTime, assertEquals(compactionRequestedTime, fileSlice.getBaseInstantTime(),
fileSlice.getBaseInstantTime()); "Ensure base-instant is now compaction request instant");
logFiles = fileSlice.getLogFiles().collect(Collectors.toList()); logFiles = fileSlice.getLogFiles().collect(Collectors.toList());
assertEquals("Only log-files after compaction request shows up", 2, logFiles.size()); assertEquals(2, logFiles.size(), "Only log-files after compaction request shows up");
assertEquals("Log File Order check", fileName4, logFiles.get(0).getFileName()); assertEquals(fileName4, logFiles.get(0).getFileName(), "Log File Order check");
assertEquals("Log File Order check", fileName3, logFiles.get(1).getFileName()); assertEquals(fileName3, logFiles.get(1).getFileName(), "Log File Order check");
// Data Files API tests // Data Files API tests
dataFiles = roView.getLatestBaseFiles().collect(Collectors.toList()); dataFiles = roView.getLatestBaseFiles().collect(Collectors.toList());
assertEquals("Expect only one data-file to be sent", 1, dataFiles.size()); assertEquals(1, dataFiles.size(), "Expect only one data-file to be sent");
dataFiles.forEach(df -> assertEquals("Expect data-file created by compaction be returned", df.getCommitTime(), compactionRequestedTime)); dataFiles.forEach(df -> assertEquals(df.getCommitTime(), compactionRequestedTime, "Expect data-file created by compaction be returned"));
dataFiles = roView.getLatestBaseFiles(partitionPath).collect(Collectors.toList()); dataFiles = roView.getLatestBaseFiles(partitionPath).collect(Collectors.toList());
assertEquals("Expect only one data-file to be sent", 1, dataFiles.size()); assertEquals(1, dataFiles.size(), "Expect only one data-file to be sent");
dataFiles.forEach(df -> assertEquals("Expect data-file created by compaction be returned", df.getCommitTime(), compactionRequestedTime)); dataFiles.forEach(df -> assertEquals(df.getCommitTime(), compactionRequestedTime, "Expect data-file created by compaction be returned"));
dataFiles = roView.getLatestBaseFilesBeforeOrOn(partitionPath, deltaInstantTime5).collect(Collectors.toList()); dataFiles = roView.getLatestBaseFilesBeforeOrOn(partitionPath, deltaInstantTime5).collect(Collectors.toList());
assertEquals("Expect only one data-file to be sent", 1, dataFiles.size()); assertEquals(1, dataFiles.size(), "Expect only one data-file to be sent");
dataFiles.forEach(df -> assertEquals("Expect data-file created by compaction be returned", df.getCommitTime(), compactionRequestedTime)); dataFiles.forEach(df -> assertEquals(df.getCommitTime(), compactionRequestedTime, "Expect data-file created by compaction be returned"));
dataFiles = roView.getLatestBaseFilesInRange(allInstantTimes).collect(Collectors.toList()); dataFiles = roView.getLatestBaseFilesInRange(allInstantTimes).collect(Collectors.toList());
assertEquals("Expect only one data-file to be sent", 1, dataFiles.size()); assertEquals(1, dataFiles.size(), "Expect only one data-file to be sent");
dataFiles.forEach(df -> assertEquals("Expect data-file created by compaction be returned", df.getCommitTime(), compactionRequestedTime)); dataFiles.forEach(df -> assertEquals(df.getCommitTime(), compactionRequestedTime, "Expect data-file created by compaction be returned"));
assertEquals("Total number of file-slices in partitions matches expected", expTotalFileSlices, assertEquals(expTotalFileSlices, rtView.getAllFileSlices(partitionPath).count(),
rtView.getAllFileSlices(partitionPath).count()); "Total number of file-slices in partitions matches expected");
assertEquals("Total number of data-files in partitions matches expected", expTotalDataFiles, assertEquals(expTotalDataFiles, roView.getAllBaseFiles(partitionPath).count(),
roView.getAllBaseFiles(partitionPath).count()); "Total number of data-files in partitions matches expected");
// file-groups includes inflight/invalid file-ids // file-groups includes inflight/invalid file-ids
assertEquals("Total number of file-groups in partitions matches expected", 5, assertEquals(5, fsView.getAllFileGroups(partitionPath).count(),
fsView.getAllFileGroups(partitionPath).count()); "Total number of file-groups in partitions matches expected");
} }
@Test @Test
@@ -533,23 +529,23 @@ public class TestHoodieTableFileSystemView extends HoodieCommonTestHarness {
new File(basePath + "/" + partitionPath).mkdirs(); new File(basePath + "/" + partitionPath).mkdirs();
String fileId = UUID.randomUUID().toString(); String fileId = UUID.randomUUID().toString();
assertFalse("No commit, should not find any data file", roView.getLatestBaseFiles(partitionPath) assertFalse(roView.getLatestBaseFiles(partitionPath).anyMatch(dfile -> dfile.getFileId().equals(fileId)),
.anyMatch(dfile -> dfile.getFileId().equals(fileId))); "No commit, should not find any data file");
// Only one commit, but is not safe // Only one commit, but is not safe
String commitTime1 = "1"; String commitTime1 = "1";
String fileName1 = FSUtils.makeDataFileName(commitTime1, TEST_WRITE_TOKEN, fileId); String fileName1 = FSUtils.makeDataFileName(commitTime1, TEST_WRITE_TOKEN, fileId);
new File(basePath + "/" + partitionPath + "/" + fileName1).createNewFile(); new File(basePath + "/" + partitionPath + "/" + fileName1).createNewFile();
refreshFsView(); refreshFsView();
assertFalse("No commit, should not find any data file", roView.getLatestBaseFiles(partitionPath) assertFalse(roView.getLatestBaseFiles(partitionPath).anyMatch(dfile -> dfile.getFileId().equals(fileId)),
.anyMatch(dfile -> dfile.getFileId().equals(fileId))); "No commit, should not find any data file");
// Make this commit safe // Make this commit safe
HoodieActiveTimeline commitTimeline = metaClient.getActiveTimeline(); HoodieActiveTimeline commitTimeline = metaClient.getActiveTimeline();
HoodieInstant instant1 = new HoodieInstant(true, HoodieTimeline.COMMIT_ACTION, commitTime1); HoodieInstant instant1 = new HoodieInstant(true, HoodieTimeline.COMMIT_ACTION, commitTime1);
saveAsComplete(commitTimeline, instant1, Option.empty()); saveAsComplete(commitTimeline, instant1, Option.empty());
refreshFsView(); refreshFsView();
assertEquals("", fileName1, roView.getLatestBaseFiles(partitionPath) assertEquals(fileName1, roView.getLatestBaseFiles(partitionPath)
.filter(dfile -> dfile.getFileId().equals(fileId)).findFirst().get().getFileName()); .filter(dfile -> dfile.getFileId().equals(fileId)).findFirst().get().getFileName());
// Do another commit, but not safe // Do another commit, but not safe
@@ -557,14 +553,14 @@ public class TestHoodieTableFileSystemView extends HoodieCommonTestHarness {
String fileName2 = FSUtils.makeDataFileName(commitTime2, TEST_WRITE_TOKEN, fileId); String fileName2 = FSUtils.makeDataFileName(commitTime2, TEST_WRITE_TOKEN, fileId);
new File(basePath + "/" + partitionPath + "/" + fileName2).createNewFile(); new File(basePath + "/" + partitionPath + "/" + fileName2).createNewFile();
refreshFsView(); refreshFsView();
assertEquals("", fileName1, roView.getLatestBaseFiles(partitionPath) assertEquals(fileName1, roView.getLatestBaseFiles(partitionPath)
.filter(dfile -> dfile.getFileId().equals(fileId)).findFirst().get().getFileName()); .filter(dfile -> dfile.getFileId().equals(fileId)).findFirst().get().getFileName());
// Make it safe // Make it safe
HoodieInstant instant2 = new HoodieInstant(true, HoodieTimeline.COMMIT_ACTION, commitTime2); HoodieInstant instant2 = new HoodieInstant(true, HoodieTimeline.COMMIT_ACTION, commitTime2);
saveAsComplete(commitTimeline, instant2, Option.empty()); saveAsComplete(commitTimeline, instant2, Option.empty());
refreshFsView(); refreshFsView();
assertEquals("", fileName2, roView.getLatestBaseFiles(partitionPath) assertEquals(fileName2, roView.getLatestBaseFiles(partitionPath)
.filter(dfile -> dfile.getFileId().equals(fileId)).findFirst().get().getFileName()); .filter(dfile -> dfile.getFileId().equals(fileId)).findFirst().get().getFileName());
} }
@@ -738,7 +734,7 @@ public class TestHoodieTableFileSystemView extends HoodieCommonTestHarness {
String fileId = fileGroup.getFileGroupId().getFileId(); String fileId = fileGroup.getFileGroupId().getFileId();
Set<String> filenames = new HashSet<>(); Set<String> filenames = new HashSet<>();
fileGroup.getAllBaseFiles().forEach(dataFile -> { fileGroup.getAllBaseFiles().forEach(dataFile -> {
assertEquals("All same fileId should be grouped", fileId, dataFile.getFileId()); assertEquals(fileId, dataFile.getFileId(), "All same fileId should be grouped");
filenames.add(dataFile.getFileName()); filenames.add(dataFile.getFileName());
}); });
Set<String> expFileNames = new HashSet<>(); Set<String> expFileNames = new HashSet<>();
@@ -1036,13 +1032,13 @@ public class TestHoodieTableFileSystemView extends HoodieCommonTestHarness {
List<HoodieFileGroup> groups = Stream.of(partitionPath1, partitionPath2, partitionPath3) List<HoodieFileGroup> groups = Stream.of(partitionPath1, partitionPath2, partitionPath3)
.flatMap(p -> fsView.getAllFileGroups(p)).collect(Collectors.toList()); .flatMap(p -> fsView.getAllFileGroups(p)).collect(Collectors.toList());
Assert.assertEquals("Expected number of file-groups", 3, groups.size()); assertEquals(3, groups.size(), "Expected number of file-groups");
Assert.assertEquals("Partitions must be different for file-groups", 3, assertEquals(3, groups.stream().map(HoodieFileGroup::getPartitionPath).collect(Collectors.toSet()).size(),
groups.stream().map(HoodieFileGroup::getPartitionPath).collect(Collectors.toSet()).size()); "Partitions must be different for file-groups");
Set<String> fileIds = groups.stream().map(HoodieFileGroup::getFileGroupId).map(HoodieFileGroupId::getFileId) Set<String> fileIds = groups.stream().map(HoodieFileGroup::getFileGroupId).map(HoodieFileGroupId::getFileId)
.collect(Collectors.toSet()); .collect(Collectors.toSet());
Assert.assertEquals("File Id must be same", 1, fileIds.size()); assertEquals(1, fileIds.size(), "File Id must be same");
Assert.assertTrue("Expected FileId", fileIds.contains(fileId)); assertTrue(fileIds.contains(fileId), "Expected FileId");
// Setup Pending compaction for all of these fileIds. // Setup Pending compaction for all of these fileIds.
List<Pair<String, FileSlice>> partitionFileSlicesPairs = new ArrayList<>(); List<Pair<String, FileSlice>> partitionFileSlicesPairs = new ArrayList<>();
@@ -1091,59 +1087,58 @@ public class TestHoodieTableFileSystemView extends HoodieCommonTestHarness {
// Test Data Files // Test Data Files
List<HoodieBaseFile> dataFiles = roView.getAllBaseFiles(partitionPath1).collect(Collectors.toList()); List<HoodieBaseFile> dataFiles = roView.getAllBaseFiles(partitionPath1).collect(Collectors.toList());
assertEquals("One data-file is expected as there is only one file-group", 1, dataFiles.size()); assertEquals(1, dataFiles.size(), "One data-file is expected as there is only one file-group");
assertEquals("Expect only valid commit", "1", dataFiles.get(0).getCommitTime()); assertEquals("1", dataFiles.get(0).getCommitTime(), "Expect only valid commit");
dataFiles = roView.getAllBaseFiles(partitionPath2).collect(Collectors.toList()); dataFiles = roView.getAllBaseFiles(partitionPath2).collect(Collectors.toList());
assertEquals("One data-file is expected as there is only one file-group", 1, dataFiles.size()); assertEquals(1, dataFiles.size(), "One data-file is expected as there is only one file-group");
assertEquals("Expect only valid commit", "1", dataFiles.get(0).getCommitTime()); assertEquals("1", dataFiles.get(0).getCommitTime(), "Expect only valid commit");
// Merge API Tests // Merge API Tests
Arrays.asList(partitionPath1, partitionPath2, partitionPath3).forEach(partitionPath -> { Arrays.asList(partitionPath1, partitionPath2, partitionPath3).forEach(partitionPath -> {
List<FileSlice> fileSliceList = List<FileSlice> fileSliceList =
rtView.getLatestMergedFileSlicesBeforeOrOn(partitionPath, deltaInstantTime5).collect(Collectors.toList()); rtView.getLatestMergedFileSlicesBeforeOrOn(partitionPath, deltaInstantTime5).collect(Collectors.toList());
assertEquals("Expect file-slice to be merged", 1, fileSliceList.size()); assertEquals(1, fileSliceList.size(), "Expect file-slice to be merged");
FileSlice fileSlice = fileSliceList.get(0); FileSlice fileSlice = fileSliceList.get(0);
assertEquals(fileId, fileSlice.getFileId()); assertEquals(fileId, fileSlice.getFileId());
assertEquals("Data file must be present", dataFileName, fileSlice.getBaseFile().get().getFileName()); assertEquals(dataFileName, fileSlice.getBaseFile().get().getFileName(), "Data file must be present");
assertEquals("Base Instant of penultimate file-slice must be base instant", instantTime1, assertEquals(instantTime1, fileSlice.getBaseInstantTime(),
fileSlice.getBaseInstantTime()); "Base Instant of penultimate file-slice must be base instant");
List<HoodieLogFile> logFiles = fileSlice.getLogFiles().collect(Collectors.toList()); List<HoodieLogFile> logFiles = fileSlice.getLogFiles().collect(Collectors.toList());
assertEquals("Log files must include those after compaction request", 3, logFiles.size()); assertEquals(3, logFiles.size(), "Log files must include those after compaction request");
assertEquals("Log File Order check", fileName4, logFiles.get(0).getFileName()); assertEquals(fileName4, logFiles.get(0).getFileName(), "Log File Order check");
assertEquals("Log File Order check", fileName3, logFiles.get(1).getFileName()); assertEquals(fileName3, logFiles.get(1).getFileName(), "Log File Order check");
assertEquals("Log File Order check", fileName1, logFiles.get(2).getFileName()); assertEquals(fileName1, logFiles.get(2).getFileName(), "Log File Order check");
fileSliceList = fileSliceList =
rtView.getLatestFileSlicesBeforeOrOn(partitionPath, deltaInstantTime5, true).collect(Collectors.toList()); rtView.getLatestFileSlicesBeforeOrOn(partitionPath, deltaInstantTime5, true).collect(Collectors.toList());
assertEquals("Expect only one file-id", 1, fileSliceList.size()); assertEquals(1, fileSliceList.size(), "Expect only one file-id");
fileSlice = fileSliceList.get(0); fileSlice = fileSliceList.get(0);
assertEquals(fileId, fileSlice.getFileId()); assertEquals(fileId, fileSlice.getFileId());
assertFalse("No data-file expected in latest file-slice", fileSlice.getBaseFile().isPresent()); assertFalse(fileSlice.getBaseFile().isPresent(), "No data-file expected in latest file-slice");
assertEquals("Compaction requested instant must be base instant", compactionRequestedTime, assertEquals(compactionRequestedTime, fileSlice.getBaseInstantTime(), "Compaction requested instant must be base instant");
fileSlice.getBaseInstantTime());
logFiles = fileSlice.getLogFiles().collect(Collectors.toList()); logFiles = fileSlice.getLogFiles().collect(Collectors.toList());
assertEquals("Log files must include only those after compaction request", 2, logFiles.size()); assertEquals(2, logFiles.size(), "Log files must include only those after compaction request");
assertEquals("Log File Order check", fileName4, logFiles.get(0).getFileName()); assertEquals(fileName4, logFiles.get(0).getFileName(), "Log File Order check");
assertEquals("Log File Order check", fileName3, logFiles.get(1).getFileName()); assertEquals(fileName3, logFiles.get(1).getFileName(), "Log File Order check");
// Check getLatestFileSlicesBeforeOrOn excluding fileIds in pending compaction // Check getLatestFileSlicesBeforeOrOn excluding fileIds in pending compaction
fileSliceList = fileSliceList =
rtView.getLatestFileSlicesBeforeOrOn(partitionPath, deltaInstantTime5, false).collect(Collectors.toList()); rtView.getLatestFileSlicesBeforeOrOn(partitionPath, deltaInstantTime5, false).collect(Collectors.toList());
assertEquals("Expect empty list as file-id is in pending compaction", 0, fileSliceList.size()); assertEquals(0, fileSliceList.size(), "Expect empty list as file-id is in pending compaction");
}); });
Assert.assertEquals(3, fsView.getPendingCompactionOperations().count()); assertEquals(3, fsView.getPendingCompactionOperations().count());
Set<String> partitionsInCompaction = fsView.getPendingCompactionOperations().map(Pair::getValue) Set<String> partitionsInCompaction = fsView.getPendingCompactionOperations().map(Pair::getValue)
.map(CompactionOperation::getPartitionPath).collect(Collectors.toSet()); .map(CompactionOperation::getPartitionPath).collect(Collectors.toSet());
Assert.assertEquals(3, partitionsInCompaction.size()); assertEquals(3, partitionsInCompaction.size());
Assert.assertTrue(partitionsInCompaction.contains(partitionPath1)); assertTrue(partitionsInCompaction.contains(partitionPath1));
Assert.assertTrue(partitionsInCompaction.contains(partitionPath2)); assertTrue(partitionsInCompaction.contains(partitionPath2));
Assert.assertTrue(partitionsInCompaction.contains(partitionPath3)); assertTrue(partitionsInCompaction.contains(partitionPath3));
Set<String> fileIdsInCompaction = fsView.getPendingCompactionOperations().map(Pair::getValue) Set<String> fileIdsInCompaction = fsView.getPendingCompactionOperations().map(Pair::getValue)
.map(CompactionOperation::getFileId).collect(Collectors.toSet()); .map(CompactionOperation::getFileId).collect(Collectors.toSet());
Assert.assertEquals(1, fileIdsInCompaction.size()); assertEquals(1, fileIdsInCompaction.size());
Assert.assertTrue(fileIdsInCompaction.contains(fileId)); assertTrue(fileIdsInCompaction.contains(fileId));
} }
private static void saveAsComplete(HoodieActiveTimeline timeline, HoodieInstant inflight, Option<byte[]> data) { private static void saveAsComplete(HoodieActiveTimeline timeline, HoodieInstant inflight, Option<byte[]> data) {

4
hudi-common/src/test/java/org/apache/hudi/common/table/view/TestRocksDbBasedFileSystemView.java
View File

@@ -21,6 +21,7 @@ package org.apache.hudi.common.table.view;
import org.apache.hudi.common.table.timeline.HoodieTimeline; import org.apache.hudi.common.table.timeline.HoodieTimeline;
import java.io.IOException; import java.io.IOException;
import java.nio.file.Files;
/** /**
* Tests RocksDB based file system view {@link SyncableFileSystemView}. * Tests RocksDB based file system view {@link SyncableFileSystemView}.
@@ -28,7 +29,8 @@ import java.io.IOException;
public class TestRocksDbBasedFileSystemView extends TestHoodieTableFileSystemView { public class TestRocksDbBasedFileSystemView extends TestHoodieTableFileSystemView {
protected SyncableFileSystemView getFileSystemView(HoodieTimeline timeline) throws IOException { protected SyncableFileSystemView getFileSystemView(HoodieTimeline timeline) throws IOException {
String subdirPath = Files.createTempDirectory(tempDir, null).toAbsolutePath().toString();
return new RocksDbBasedFileSystemView(metaClient, timeline, return new RocksDbBasedFileSystemView(metaClient, timeline,
FileSystemViewStorageConfig.newBuilder().withRocksDBPath(folder.newFolder().getAbsolutePath()).build()); FileSystemViewStorageConfig.newBuilder().withRocksDBPath(subdirPath).build());
} }
} }

52
hudi-common/src/test/java/org/apache/hudi/common/testutils/HoodieCommonTestHarnessJunit5.java Normal file
View File

@@ -0,0 +1,52 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hudi.common.testutils;
import org.apache.hudi.common.model.HoodieTestUtils;
import org.apache.hudi.common.table.HoodieTableMetaClient;
import org.junit.jupiter.api.io.TempDir;
import java.io.IOException;
/**
* The JUnit 5 version of {@link org.apache.hudi.common.HoodieCommonTestHarness}.
* <p>
* To incrementally migrate test classes.
*/
public class HoodieCommonTestHarnessJunit5 extends org.apache.hudi.common.HoodieCommonTestHarness {
@TempDir
public java.nio.file.Path tempDir;
/**
* Initializes basePath.
*/
protected void initPath() {
this.basePath = tempDir.toAbsolutePath().toString();
}
/**
* Initializes an instance of {@link HoodieTableMetaClient} with a special table type specified by {@code getTableType()}.
*/
protected void initMetaClient() throws IOException {
metaClient = HoodieTestUtils.init(tempDir.toAbsolutePath().toString(), getTableType());
basePath = metaClient.getBasePath();
}
}

20
hudi-common/src/test/java/org/apache/hudi/common/util/TestFileIOUtils.java
View File

@@ -18,39 +18,39 @@
package org.apache.hudi.common.util; package org.apache.hudi.common.util;
import org.apache.hudi.common.HoodieCommonTestHarness; import org.apache.hudi.common.testutils.HoodieCommonTestHarnessJunit5;
import org.junit.Test; import org.junit.jupiter.api.Test;
import java.io.ByteArrayInputStream; import java.io.ByteArrayInputStream;
import java.io.File; import java.io.File;
import java.io.IOException; import java.io.IOException;
import java.nio.charset.StandardCharsets; import java.nio.charset.StandardCharsets;
import static org.junit.Assert.assertEquals; import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.Assert.assertFalse; import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.Assert.assertTrue; import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.junit.Assert.fail; import static org.junit.jupiter.api.Assertions.fail;
/** /**
* Tests file I/O utils. * Tests file I/O utils.
*/ */
public class TestFileIOUtils extends HoodieCommonTestHarness { public class TestFileIOUtils extends HoodieCommonTestHarnessJunit5 {
@Test @Test
public void testMkdirAndDelete() throws IOException { public void testMkdirAndDelete() throws IOException {
try { try {
FileIOUtils.mkdir(folder.getRoot()); FileIOUtils.mkdir(tempDir.toFile());
} catch (IOException e) { } catch (IOException e) {
fail("Should not error out if dir exists already"); fail("Should not error out if dir exists already");
} }
File dir = new File(folder.getRoot().getAbsolutePath() + "/dir"); File dir = tempDir.resolve("dir").toFile();
FileIOUtils.mkdir(dir); FileIOUtils.mkdir(dir);
assertTrue(dir.exists()); assertTrue(dir.exists());
new File(dir, "t.txt").createNewFile(); new File(dir, "t.txt").createNewFile();
new File(dir, "subdir").mkdirs(); new File(dir, "subdir").mkdirs();
new File(dir, "subdir/z.txt").createNewFile(); new File(dir, "subdir" + File.pathSeparator + "z.txt").createNewFile();
FileIOUtils.deleteDirectory(dir); FileIOUtils.deleteDirectory(dir);
assertFalse(dir.exists()); assertFalse(dir.exists());
} }

6
hudi-hadoop-mr/pom.xml
View File

@@ -96,12 +96,6 @@
<scope>test</scope> <scope>test</scope>
</dependency> </dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<scope>test</scope>
</dependency>
<dependency> <dependency>
<groupId>org.junit.jupiter</groupId> <groupId>org.junit.jupiter</groupId>
<artifactId>junit-jupiter-api</artifactId> <artifactId>junit-jupiter-api</artifactId>

65
hudi-hadoop-mr/src/test/java/org/apache/hudi/hadoop/InputFormatTestUtil.java
View File

@@ -39,10 +39,11 @@ import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants;
import org.apache.hadoop.hive.serde2.ColumnProjectionUtils; import org.apache.hadoop.hive.serde2.ColumnProjectionUtils;
import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.mapred.JobConf;
import org.apache.parquet.avro.AvroParquetWriter; import org.apache.parquet.avro.AvroParquetWriter;
import org.junit.rules.TemporaryFolder;
import java.io.File; import java.io.File;
import java.io.IOException; import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Arrays; import java.util.Arrays;
import java.util.Collections; import java.util.Collections;
@@ -57,19 +58,19 @@ public class InputFormatTestUtil {
private static String TEST_WRITE_TOKEN = "1-0-1"; private static String TEST_WRITE_TOKEN = "1-0-1";
public static File prepareTable(TemporaryFolder basePath, int numberOfFiles, String commitNumber) public static File prepareTable(java.nio.file.Path basePath, int numberOfFiles, String commitNumber)
throws IOException { throws IOException {
basePath.create(); HoodieTestUtils.init(HoodieTestUtils.getDefaultHadoopConf(), basePath.toString());
HoodieTestUtils.init(HoodieTestUtils.getDefaultHadoopConf(), basePath.getRoot().toString()); java.nio.file.Path partitionPath = basePath.resolve(Paths.get("2016", "05", "01"));
File partitionPath = basePath.newFolder("2016", "05", "01"); Files.createDirectories(partitionPath);
return simulateInserts(partitionPath, "fileId1", numberOfFiles, commitNumber); return simulateInserts(partitionPath.toFile(), "fileId1", numberOfFiles, commitNumber);
} }
public static File simulateInserts(File partitionPath, String fileId, int numberOfFiles, String commitNumber) public static File simulateInserts(File partitionPath, String fileId, int numberOfFiles, String commitNumber)
throws IOException { throws IOException {
for (int i = 0; i < numberOfFiles; i++) { for (int i = 0; i < numberOfFiles; i++) {
File dataFile = new File(partitionPath, FSUtils.makeDataFileName(commitNumber, TEST_WRITE_TOKEN, fileId + i)); Files.createFile(partitionPath.toPath()
dataFile.createNewFile(); .resolve(FSUtils.makeDataFileName(commitNumber, TEST_WRITE_TOKEN, fileId + i)));
} }
return partitionPath; return partitionPath;
} }
@@ -86,19 +87,18 @@ public class InputFormatTestUtil {
List<File> toUpdateList = dataFiles.subList(0, Math.min(numberOfFilesUpdated, dataFiles.size())); List<File> toUpdateList = dataFiles.subList(0, Math.min(numberOfFilesUpdated, dataFiles.size()));
for (File file : toUpdateList) { for (File file : toUpdateList) {
String fileId = FSUtils.getFileId(file.getName()); String fileId = FSUtils.getFileId(file.getName());
File dataFile = new File(directory, FSUtils.makeDataFileName(newCommit, TEST_WRITE_TOKEN, fileId)); Files.createFile(directory.toPath().resolve(FSUtils.makeDataFileName(newCommit, TEST_WRITE_TOKEN, fileId)));
dataFile.createNewFile();
} }
} }
public static void commit(TemporaryFolder basePath, String commitNumber) throws IOException { public static void commit(java.nio.file.Path basePath, String commitNumber) throws IOException {
// create the commit // create the commit
new File(basePath.getRoot().toString() + "/.hoodie/", commitNumber + ".commit").createNewFile(); Files.createFile(basePath.resolve(Paths.get(".hoodie", commitNumber + ".commit")));
} }
public static void deltaCommit(TemporaryFolder basePath, String commitNumber) throws IOException { public static void deltaCommit(java.nio.file.Path basePath, String commitNumber) throws IOException {
// create the commit // create the commit
new File(basePath.getRoot().toString() + "/.hoodie/", commitNumber + ".deltacommit").createNewFile(); Files.createFile(basePath.resolve(Paths.get(".hoodie", commitNumber + ".deltacommit")));
} }
public static void setupIncremental(JobConf jobConf, String startCommit, int numberOfCommitsToPull) { public static void setupIncremental(JobConf jobConf, String startCommit, int numberOfCommitsToPull) {
@@ -119,40 +119,35 @@ public class InputFormatTestUtil {
return new Schema.Parser().parse(InputFormatTestUtil.class.getResourceAsStream(location)); return new Schema.Parser().parse(InputFormatTestUtil.class.getResourceAsStream(location));
} }
public static File prepareParquetTable(TemporaryFolder basePath, Schema schema, int numberOfFiles, public static File prepareParquetTable(java.nio.file.Path basePath, Schema schema, int numberOfFiles,
int numberOfRecords, String commitNumber) throws IOException { int numberOfRecords, String commitNumber) throws IOException {
basePath.create(); HoodieTestUtils.init(HoodieTestUtils.getDefaultHadoopConf(), basePath.toString());
HoodieTestUtils.init(HoodieTestUtils.getDefaultHadoopConf(), basePath.getRoot().toString()); java.nio.file.Path partitionPath = basePath.resolve(Paths.get("2016", "05", "01"));
File partitionPath = basePath.newFolder("2016", "05", "01");
createData(schema, partitionPath, numberOfFiles, numberOfRecords, commitNumber); createData(schema, partitionPath, numberOfFiles, numberOfRecords, commitNumber);
return partitionPath; return partitionPath.toFile();
} }
public static File prepareSimpleParquetTable(TemporaryFolder basePath, Schema schema, int numberOfFiles, public static File prepareSimpleParquetTable(java.nio.file.Path basePath, Schema schema, int numberOfFiles,
int numberOfRecords, String commitNumber) throws Exception { int numberOfRecords, String commitNumber) throws Exception {
basePath.create(); HoodieTestUtils.init(HoodieTestUtils.getDefaultHadoopConf(), basePath.toString());
HoodieTestUtils.init(HoodieTestUtils.getDefaultHadoopConf(), basePath.getRoot().toString()); java.nio.file.Path partitionPath = basePath.resolve(Paths.get("2016", "05", "01"));
File partitionPath = basePath.newFolder("2016", "05", "01");
createSimpleData(schema, partitionPath, numberOfFiles, numberOfRecords, commitNumber); createSimpleData(schema, partitionPath, numberOfFiles, numberOfRecords, commitNumber);
return partitionPath; return partitionPath.toFile();
} }
public static File prepareNonPartitionedParquetTable(TemporaryFolder baseDir, Schema schema, int numberOfFiles, public static File prepareNonPartitionedParquetTable(java.nio.file.Path basePath, Schema schema, int numberOfFiles,
int numberOfRecords, String commitNumber) throws IOException { int numberOfRecords, String commitNumber) throws IOException {
baseDir.create(); HoodieTestUtils.init(HoodieTestUtils.getDefaultHadoopConf(), basePath.toString());
HoodieTestUtils.init(HoodieTestUtils.getDefaultHadoopConf(), baseDir.getRoot().toString());
File basePath = baseDir.getRoot();
createData(schema, basePath, numberOfFiles, numberOfRecords, commitNumber); createData(schema, basePath, numberOfFiles, numberOfRecords, commitNumber);
return basePath; return basePath.toFile();
} }
private static void createData(Schema schema, File partitionPath, int numberOfFiles, int numberOfRecords, private static void createData(Schema schema, java.nio.file.Path partitionPath, int numberOfFiles, int numberOfRecords,
String commitNumber) throws IOException { String commitNumber) throws IOException {
AvroParquetWriter parquetWriter; AvroParquetWriter parquetWriter;
for (int i = 0; i < numberOfFiles; i++) { for (int i = 0; i < numberOfFiles; i++) {
String fileId = FSUtils.makeDataFileName(commitNumber, TEST_WRITE_TOKEN, "fileid" + i); String fileId = FSUtils.makeDataFileName(commitNumber, TEST_WRITE_TOKEN, "fileid" + i);
File dataFile = new File(partitionPath, fileId); parquetWriter = new AvroParquetWriter(new Path(partitionPath.resolve(fileId).toString()), schema);
parquetWriter = new AvroParquetWriter(new Path(dataFile.getAbsolutePath()), schema);
try { try {
for (GenericRecord record : generateAvroRecords(schema, numberOfRecords, commitNumber, fileId)) { for (GenericRecord record : generateAvroRecords(schema, numberOfRecords, commitNumber, fileId)) {
parquetWriter.write(record); parquetWriter.write(record);
@@ -163,13 +158,12 @@ public class InputFormatTestUtil {
} }
} }
private static void createSimpleData(Schema schema, File partitionPath, int numberOfFiles, int numberOfRecords, private static void createSimpleData(Schema schema, java.nio.file.Path partitionPath, int numberOfFiles, int numberOfRecords,
String commitNumber) throws Exception { String commitNumber) throws Exception {
AvroParquetWriter parquetWriter; AvroParquetWriter parquetWriter;
for (int i = 0; i < numberOfFiles; i++) { for (int i = 0; i < numberOfFiles; i++) {
String fileId = FSUtils.makeDataFileName(commitNumber, "1", "fileid" + i); String fileId = FSUtils.makeDataFileName(commitNumber, "1", "fileid" + i);
File dataFile = new File(partitionPath, fileId); parquetWriter = new AvroParquetWriter(new Path(partitionPath.resolve(fileId).toString()), schema);
parquetWriter = new AvroParquetWriter(new Path(dataFile.getAbsolutePath()), schema);
try { try {
List<IndexedRecord> records = SchemaTestUtil.generateTestRecords(0, numberOfRecords); List<IndexedRecord> records = SchemaTestUtil.generateTestRecords(0, numberOfRecords);
Schema hoodieFieldsSchema = HoodieAvroUtils.addMetadataFields(schema); Schema hoodieFieldsSchema = HoodieAvroUtils.addMetadataFields(schema);
@@ -212,7 +206,6 @@ public class InputFormatTestUtil {
parquetWriter.write(record); parquetWriter.write(record);
} }
} }
} }
public static HoodieLogFormat.Writer writeRollback(File partitionDir, FileSystem fs, String fileId, String baseCommit, public static HoodieLogFormat.Writer writeRollback(File partitionDir, FileSystem fs, String fileId, String baseCommit,

71
hudi-hadoop-mr/src/test/java/org/apache/hudi/hadoop/TestHoodieParquetInputFormat.java
View File

@@ -23,6 +23,11 @@ import org.apache.hudi.common.fs.FSUtils;
import org.apache.hudi.common.model.HoodieCommitMetadata; import org.apache.hudi.common.model.HoodieCommitMetadata;
import org.apache.hudi.common.model.HoodieTestUtils; import org.apache.hudi.common.model.HoodieTestUtils;
import org.apache.hudi.common.model.HoodieWriteStat; import org.apache.hudi.common.model.HoodieWriteStat;
import org.apache.hudi.common.table.HoodieTableMetaClient;
import org.apache.hudi.common.table.timeline.HoodieActiveTimeline;
import org.apache.hudi.common.table.timeline.HoodieInstant;
import org.apache.hudi.common.table.timeline.HoodieTimeline;
import org.apache.hudi.common.table.timeline.TimelineMetadataUtils;
import org.apache.avro.Schema; import org.apache.avro.Schema;
import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileStatus;
@@ -33,15 +38,9 @@ import org.apache.hadoop.mapred.InputSplit;
import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.RecordReader; import org.apache.hadoop.mapred.RecordReader;
import org.apache.hadoop.mapreduce.Job; import org.apache.hadoop.mapreduce.Job;
import org.apache.hudi.common.table.HoodieTableMetaClient; import org.junit.jupiter.api.BeforeEach;
import org.apache.hudi.common.table.timeline.HoodieActiveTimeline; import org.junit.jupiter.api.Test;
import org.apache.hudi.common.table.timeline.HoodieInstant; import org.junit.jupiter.api.io.TempDir;
import org.apache.hudi.common.table.timeline.HoodieTimeline;
import org.apache.hudi.common.table.timeline.TimelineMetadataUtils;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
import java.io.File; import java.io.File;
import java.io.FileOutputStream; import java.io.FileOutputStream;
@@ -50,15 +49,12 @@ import java.nio.charset.StandardCharsets;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.List;
import static org.junit.Assert.assertEquals; import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.Assert.assertFalse; import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.Assert.assertTrue; import static org.junit.jupiter.api.Assertions.assertTrue;
public class TestHoodieParquetInputFormat { public class TestHoodieParquetInputFormat {
@Rule
public TemporaryFolder basePath = new TemporaryFolder();
private HoodieParquetInputFormat inputFormat; private HoodieParquetInputFormat inputFormat;
private JobConf jobConf; private JobConf jobConf;
@@ -70,16 +66,19 @@ public class TestHoodieParquetInputFormat {
count++; count++;
} }
} }
assertEquals(msg, expected, count); assertEquals(expected, count, msg);
} }
@Before @BeforeEach
public void setUp() { public void setUp() {
inputFormat = new HoodieParquetInputFormat(); inputFormat = new HoodieParquetInputFormat();
jobConf = new JobConf(); jobConf = new JobConf();
inputFormat.setConf(jobConf); inputFormat.setConf(jobConf);
} }
@TempDir
public java.nio.file.Path basePath;
// Verify that HoodieParquetInputFormat does not return instants after pending compaction // Verify that HoodieParquetInputFormat does not return instants after pending compaction
@Test @Test
public void testPendingCompactionWithActiveCommits() throws IOException { public void testPendingCompactionWithActiveCommits() throws IOException {
@@ -98,7 +97,7 @@ public class TestHoodieParquetInputFormat {
instants.add(t4); instants.add(t4);
instants.add(t5); instants.add(t5);
instants.add(t6); instants.add(t6);
HoodieTableMetaClient metaClient = HoodieTestUtils.init(basePath.getRoot().getAbsolutePath().toString()); HoodieTableMetaClient metaClient = HoodieTestUtils.init(basePath.toString());
HoodieActiveTimeline timeline = new HoodieActiveTimeline(metaClient); HoodieActiveTimeline timeline = new HoodieActiveTimeline(metaClient);
timeline.setInstants(instants); timeline.setInstants(instants);
@@ -196,16 +195,16 @@ public class TestHoodieParquetInputFormat {
InputFormatTestUtil.setupIncremental(jobConf, "100", 1); InputFormatTestUtil.setupIncremental(jobConf, "100", 1);
FileStatus[] files = inputFormat.listStatus(jobConf); FileStatus[] files = inputFormat.listStatus(jobConf);
assertEquals("We should exclude commit 100 when returning incremental pull with start commit time as 100", 0, assertEquals(0, files.length,
files.length); "We should exclude commit 100 when returning incremental pull with start commit time as 100");
} }
private void createCommitFile(TemporaryFolder basePath, String commitNumber, String partitionPath) private void createCommitFile(java.nio.file.Path basePath, String commitNumber, String partitionPath)
throws IOException { throws IOException {
List<HoodieWriteStat> writeStats = HoodieTestUtils.generateFakeHoodieWriteStat(1); List<HoodieWriteStat> writeStats = HoodieTestUtils.generateFakeHoodieWriteStat(1);
HoodieCommitMetadata commitMetadata = new HoodieCommitMetadata(); HoodieCommitMetadata commitMetadata = new HoodieCommitMetadata();
writeStats.forEach(stat -> commitMetadata.addWriteStat(partitionPath, stat)); writeStats.forEach(stat -> commitMetadata.addWriteStat(partitionPath, stat));
File file = new File(basePath.getRoot().toString() + "/.hoodie/", commitNumber + ".commit"); File file = basePath.resolve(".hoodie").resolve(commitNumber + ".commit").toFile();
file.createNewFile(); file.createNewFile();
FileOutputStream fileOutputStream = new FileOutputStream(file); FileOutputStream fileOutputStream = new FileOutputStream(file);
fileOutputStream.write(commitMetadata.toJsonString().getBytes(StandardCharsets.UTF_8)); fileOutputStream.write(commitMetadata.toJsonString().getBytes(StandardCharsets.UTF_8));
@@ -213,10 +212,10 @@ public class TestHoodieParquetInputFormat {
fileOutputStream.close(); fileOutputStream.close();
} }
private File createCompactionFile(TemporaryFolder basePath, String commitTime) private File createCompactionFile(java.nio.file.Path basePath, String commitTime)
throws IOException { throws IOException {
File file = new File(basePath.getRoot().toString() + "/.hoodie/", File file = basePath.resolve(".hoodie")
HoodieTimeline.makeRequestedCompactionFileName(commitTime)); .resolve(HoodieTimeline.makeRequestedCompactionFileName(commitTime)).toFile();
assertTrue(file.createNewFile()); assertTrue(file.createNewFile());
FileOutputStream os = new FileOutputStream(file); FileOutputStream os = new FileOutputStream(file);
try { try {
@@ -255,14 +254,14 @@ public class TestHoodieParquetInputFormat {
InputFormatTestUtil.setupIncremental(jobConf, "100", 1); InputFormatTestUtil.setupIncremental(jobConf, "100", 1);
FileStatus[] files = inputFormat.listStatus(jobConf); FileStatus[] files = inputFormat.listStatus(jobConf);
assertEquals("Pulling 1 commit from 100, should get us the 5 files committed at 200", 5, files.length); assertEquals(5, files.length, "Pulling 1 commit from 100, should get us the 5 files committed at 200");
ensureFilesInCommit("Pulling 1 commit from 100, should get us the 5 files committed at 200", files, "200", 5); ensureFilesInCommit("Pulling 1 commit from 100, should get us the 5 files committed at 200", files, "200", 5);
InputFormatTestUtil.setupIncremental(jobConf, "100", 3); InputFormatTestUtil.setupIncremental(jobConf, "100", 3);
files = inputFormat.listStatus(jobConf); files = inputFormat.listStatus(jobConf);
assertEquals("Pulling 3 commits from 100, should get us the 3 files from 400 commit, 1 file from 300 " assertEquals(5, files.length, "Pulling 3 commits from 100, should get us the 3 files from 400 commit, 1 file from 300 "
+ "commit and 1 file from 200 commit", 5, files.length); + "commit and 1 file from 200 commit");
ensureFilesInCommit("Pulling 3 commits from 100, should get us the 3 files from 400 commit", files, "400", 3); ensureFilesInCommit("Pulling 3 commits from 100, should get us the 3 files from 400 commit", files, "400", 3);
ensureFilesInCommit("Pulling 3 commits from 100, should get us the 1 files from 300 commit", files, "300", 1); ensureFilesInCommit("Pulling 3 commits from 100, should get us the 1 files from 300 commit", files, "300", 1);
ensureFilesInCommit("Pulling 3 commits from 100, should get us the 1 files from 200 commit", files, "200", 1); ensureFilesInCommit("Pulling 3 commits from 100, should get us the 1 files from 200 commit", files, "200", 1);
@@ -270,8 +269,8 @@ public class TestHoodieParquetInputFormat {
InputFormatTestUtil.setupIncremental(jobConf, "100", HoodieHiveUtil.MAX_COMMIT_ALL); InputFormatTestUtil.setupIncremental(jobConf, "100", HoodieHiveUtil.MAX_COMMIT_ALL);
files = inputFormat.listStatus(jobConf); files = inputFormat.listStatus(jobConf);
assertEquals("Pulling all commits from 100, should get us the 1 file from each of 200,300,400,500,400 commits", assertEquals(5, files.length,
5, files.length); "Pulling all commits from 100, should get us the 1 file from each of 200,300,400,500,400 commits");
ensureFilesInCommit("Pulling all commits from 100, should get us the 1 files from 600 commit", files, "600", 1); ensureFilesInCommit("Pulling all commits from 100, should get us the 1 files from 600 commit", files, "600", 1);
ensureFilesInCommit("Pulling all commits from 100, should get us the 1 files from 500 commit", files, "500", 1); ensureFilesInCommit("Pulling all commits from 100, should get us the 1 files from 500 commit", files, "500", 1);
ensureFilesInCommit("Pulling all commits from 100, should get us the 1 files from 400 commit", files, "400", 1); ensureFilesInCommit("Pulling all commits from 100, should get us the 1 files from 400 commit", files, "400", 1);
@@ -342,8 +341,8 @@ public class TestHoodieParquetInputFormat {
FileInputFormat.setInputPaths(jobConf, partitionDir.getPath()); FileInputFormat.setInputPaths(jobConf, partitionDir.getPath());
InputFormatTestUtil.setupIncremental(jobConf, "0", -1); InputFormatTestUtil.setupIncremental(jobConf, "0", -1);
FileStatus[] files = inputFormat.listStatus(jobConf); FileStatus[] files = inputFormat.listStatus(jobConf);
assertEquals("Pulling all commit from beginning, should not return instants after begin compaction", assertEquals(10, files.length,
10, files.length); "Pulling all commit from beginning, should not return instants after begin compaction");
ensureFilesInCommit("Pulling all commit from beginning, should not return instants after begin compaction", ensureFilesInCommit("Pulling all commit from beginning, should not return instants after begin compaction",
files, "100", 10); files, "100", 10);
@@ -351,8 +350,8 @@ public class TestHoodieParquetInputFormat {
compactionFile.delete(); compactionFile.delete();
InputFormatTestUtil.setupIncremental(jobConf, "0", -1); InputFormatTestUtil.setupIncremental(jobConf, "0", -1);
files = inputFormat.listStatus(jobConf); files = inputFormat.listStatus(jobConf);
assertEquals("after deleting compaction, should get all inserted files", assertEquals(20, files.length,
20, files.length); "after deleting compaction, should get all inserted files");
ensureFilesInCommit("Pulling all commit from beginning, should return instants before requested compaction", ensureFilesInCommit("Pulling all commit from beginning, should return instants before requested compaction",
files, "100", 10); files, "100", 10);
@@ -381,7 +380,7 @@ public class TestHoodieParquetInputFormat {
totalCount++; totalCount++;
} }
} }
assertEquals(msg, expectedNumberOfRecordsInCommit, actualCount); assertEquals(expectedNumberOfRecordsInCommit, actualCount, msg);
assertEquals(msg, totalExpected, totalCount); assertEquals(totalExpected, totalCount, msg);
} }
} }

26
hudi-hadoop-mr/src/test/java/org/apache/hudi/hadoop/TestHoodieROTablePathFilter.java
View File

@@ -18,28 +18,28 @@
package org.apache.hudi.hadoop; package org.apache.hudi.hadoop;
import org.apache.hudi.common.HoodieCommonTestHarness;
import org.apache.hudi.common.model.HoodieTestUtils; import org.apache.hudi.common.model.HoodieTestUtils;
import org.apache.hudi.common.table.HoodieTableMetaClient; import org.apache.hudi.common.table.HoodieTableMetaClient;
import org.apache.hudi.common.testutils.HoodieCommonTestHarnessJunit5;
import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.Path;
import org.junit.Before; import org.junit.jupiter.api.BeforeEach;
import org.junit.Test; import org.junit.jupiter.api.Test;
import org.junit.rules.TemporaryFolder; import org.junit.jupiter.api.io.TempDir;
import java.io.File; import java.io.File;
import java.io.IOException; import java.io.IOException;
import java.util.ArrayList; import java.util.ArrayList;
import static org.junit.Assert.assertFalse; import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.Assert.assertTrue; import static org.junit.jupiter.api.Assertions.assertTrue;
/** /**
* *
*/ */
public class TestHoodieROTablePathFilter extends HoodieCommonTestHarness { public class TestHoodieROTablePathFilter extends HoodieCommonTestHarnessJunit5 {
@Before @BeforeEach
public void setUp() throws Exception { public void setUp() throws Exception {
initMetaClient(); initMetaClient();
} }
@@ -61,7 +61,7 @@ public class TestHoodieROTablePathFilter extends HoodieCommonTestHarness {
HoodieROTablePathFilter pathFilter = new HoodieROTablePathFilter(); HoodieROTablePathFilter pathFilter = new HoodieROTablePathFilter();
Path partitionPath = new Path("file://" + basePath + File.separator + "2017/01/01"); Path partitionPath = new Path("file://" + basePath + File.separator + "2017/01/01");
assertTrue("Directories should be accepted", pathFilter.accept(partitionPath)); assertTrue(pathFilter.accept(partitionPath), "Directories should be accepted");
assertTrue( assertTrue(
pathFilter.accept(new Path("file:///" + HoodieTestUtils.getDataFilePath(basePath, "2017/01/01", "001", "f1")))); pathFilter.accept(new Path("file:///" + HoodieTestUtils.getDataFilePath(basePath, "2017/01/01", "001", "f1"))));
@@ -87,10 +87,8 @@ public class TestHoodieROTablePathFilter extends HoodieCommonTestHarness {
} }
@Test @Test
public void testNonHoodiePaths() throws IOException { public void testNonHoodiePaths(@TempDir java.nio.file.Path tempDir) throws IOException {
TemporaryFolder folder = new TemporaryFolder(); String basePath = tempDir.toAbsolutePath().toString();
folder.create();
String basePath = folder.getRoot().getAbsolutePath();
HoodieROTablePathFilter pathFilter = new HoodieROTablePathFilter(); HoodieROTablePathFilter pathFilter = new HoodieROTablePathFilter();
String path = basePath + File.separator + "nonhoodiefolder"; String path = basePath + File.separator + "nonhoodiefolder";
@@ -100,7 +98,5 @@ public class TestHoodieROTablePathFilter extends HoodieCommonTestHarness {
path = basePath + File.separator + "nonhoodiefolder/somefile"; path = basePath + File.separator + "nonhoodiefolder/somefile";
new File(path).createNewFile(); new File(path).createNewFile();
assertTrue(pathFilter.accept(new Path("file:///" + path))); assertTrue(pathFilter.accept(new Path("file:///" + path)));
folder.delete();
} }
} }

52
hudi-hadoop-mr/src/test/java/org/apache/hudi/hadoop/realtime/TestHoodieCombineHiveInputFormat.java
View File

@@ -18,19 +18,15 @@
package org.apache.hudi.hadoop.realtime; package org.apache.hudi.hadoop.realtime;
import static org.apache.hadoop.hive.ql.exec.Utilities.HAS_MAP_WORK;
import static org.apache.hadoop.hive.ql.exec.Utilities.MAPRED_MAPPER_CLASS;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import org.apache.hudi.avro.HoodieAvroUtils; import org.apache.hudi.avro.HoodieAvroUtils;
import org.apache.hudi.common.HoodieCommonTestHarness;
import org.apache.hudi.common.minicluster.MiniClusterUtil; import org.apache.hudi.common.minicluster.MiniClusterUtil;
import org.apache.hudi.common.model.HoodieTableType; import org.apache.hudi.common.model.HoodieTableType;
import org.apache.hudi.common.model.HoodieTestUtils; import org.apache.hudi.common.model.HoodieTestUtils;
import org.apache.hudi.common.table.log.HoodieLogFormat; import org.apache.hudi.common.table.log.HoodieLogFormat;
import org.apache.hudi.common.testutils.HoodieCommonTestHarnessJunit5;
import org.apache.hudi.common.util.SchemaTestUtil; import org.apache.hudi.common.util.SchemaTestUtil;
import org.apache.hudi.hadoop.InputFormatTestUtil; import org.apache.hudi.hadoop.InputFormatTestUtil;
import org.apache.hudi.hadoop.hive.HoodieCombineHiveInputFormat;
import org.apache.avro.Schema; import org.apache.avro.Schema;
import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configuration;
@@ -47,60 +43,60 @@ import org.apache.hadoop.mapred.FileInputFormat;
import org.apache.hadoop.mapred.InputSplit; import org.apache.hadoop.mapred.InputSplit;
import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.RecordReader; import org.apache.hadoop.mapred.RecordReader;
import org.apache.hudi.hadoop.hive.HoodieCombineHiveInputFormat; import org.junit.jupiter.api.AfterAll;
import org.junit.AfterClass; import org.junit.jupiter.api.BeforeAll;
import org.junit.Before; import org.junit.jupiter.api.BeforeEach;
import org.junit.BeforeClass; import org.junit.jupiter.api.Disabled;
import org.junit.Ignore; import org.junit.jupiter.api.Test;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
import java.io.File; import java.io.File;
import java.io.IOException; import java.io.IOException;
import java.util.LinkedHashMap; import java.util.LinkedHashMap;
public class TestHoodieCombineHiveInputFormat extends HoodieCommonTestHarness { import static org.apache.hadoop.hive.ql.exec.Utilities.HAS_MAP_WORK;
import static org.apache.hadoop.hive.ql.exec.Utilities.MAPRED_MAPPER_CLASS;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
public class TestHoodieCombineHiveInputFormat extends HoodieCommonTestHarnessJunit5 {
@Rule
public TemporaryFolder basePath = new TemporaryFolder();
private JobConf jobConf; private JobConf jobConf;
private FileSystem fs; private FileSystem fs;
private Configuration hadoopConf; private Configuration hadoopConf;
@BeforeClass @BeforeAll
public static void setUpClass() throws IOException, InterruptedException { public static void setUpClass() throws IOException, InterruptedException {
// Append is not supported in LocalFileSystem. HDFS needs to be setup. // Append is not supported in LocalFileSystem. HDFS needs to be setup.
MiniClusterUtil.setUp(); MiniClusterUtil.setUp();
} }
@AfterClass @AfterAll
public static void tearDownClass() { public static void tearDownClass() {
MiniClusterUtil.shutdown(); MiniClusterUtil.shutdown();
} }
@Before @BeforeEach
public void setUp() throws IOException, InterruptedException { public void setUp() throws IOException, InterruptedException {
this.fs = MiniClusterUtil.fileSystem; this.fs = MiniClusterUtil.fileSystem;
jobConf = new JobConf(); jobConf = new JobConf();
hadoopConf = HoodieTestUtils.getDefaultHadoopConf(); hadoopConf = HoodieTestUtils.getDefaultHadoopConf();
assertTrue(fs.mkdirs(new Path(folder.getRoot().getPath()))); assertTrue(fs.mkdirs(new Path(folder.getRoot().getPath())));
HoodieTestUtils.init(MiniClusterUtil.configuration, basePath.getRoot().getPath(), HoodieTableType.MERGE_ON_READ); HoodieTestUtils.init(MiniClusterUtil.configuration, tempDir.toAbsolutePath().toString(), HoodieTableType.MERGE_ON_READ);
} }
@Test @Test
@Ignore @Disabled
public void testHoodieRealtimeCombineHoodieInputFormat() throws Exception { public void testHoodieRealtimeCombineHoodieInputFormat() throws Exception {
Configuration conf = new Configuration(); Configuration conf = new Configuration();
// initial commit // initial commit
Schema schema = HoodieAvroUtils.addMetadataFields(SchemaTestUtil.getEvolvedSchema()); Schema schema = HoodieAvroUtils.addMetadataFields(SchemaTestUtil.getEvolvedSchema());
HoodieTestUtils.init(hadoopConf, basePath.getRoot().getAbsolutePath(), HoodieTableType.MERGE_ON_READ); HoodieTestUtils.init(hadoopConf, tempDir.toAbsolutePath().toString(), HoodieTableType.MERGE_ON_READ);
String commitTime = "100"; String commitTime = "100";
final int numRecords = 1000; final int numRecords = 1000;
// Create 3 parquet files with 1000 records each // Create 3 parquet files with 1000 records each
File partitionDir = InputFormatTestUtil.prepareParquetTable(basePath, schema, 3, numRecords, commitTime); File partitionDir = InputFormatTestUtil.prepareParquetTable(tempDir, schema, 3, numRecords, commitTime);
InputFormatTestUtil.commit(basePath, commitTime); InputFormatTestUtil.commit(tempDir, commitTime);
// insert 1000 update records to log file 0 // insert 1000 update records to log file 0
String newCommitTime = "101"; String newCommitTime = "101";
@@ -124,10 +120,10 @@ public class TestHoodieCombineHiveInputFormat extends HoodieCommonTestHarness {
tblDesc.setInputFileFormatClass(HoodieCombineHiveInputFormat.class); tblDesc.setInputFileFormatClass(HoodieCombineHiveInputFormat.class);
PartitionDesc partDesc = new PartitionDesc(tblDesc, null); PartitionDesc partDesc = new PartitionDesc(tblDesc, null);
LinkedHashMap<Path, PartitionDesc> pt = new LinkedHashMap<>(); LinkedHashMap<Path, PartitionDesc> pt = new LinkedHashMap<>();
pt.put(new Path(basePath.getRoot().getAbsolutePath()), partDesc); pt.put(new Path(tempDir.toAbsolutePath().toString()), partDesc);
MapredWork mrwork = new MapredWork(); MapredWork mrwork = new MapredWork();
mrwork.getMapWork().setPathToPartitionInfo(pt); mrwork.getMapWork().setPathToPartitionInfo(pt);
Path mapWorkPath = new Path(basePath.getRoot().getAbsolutePath()); Path mapWorkPath = new Path(tempDir.toAbsolutePath().toString());
Utilities.setMapRedWork(conf, mrwork, mapWorkPath); Utilities.setMapRedWork(conf, mrwork, mapWorkPath);
jobConf = new JobConf(conf); jobConf = new JobConf(conf);
// Add the paths // Add the paths
@@ -143,7 +139,7 @@ public class TestHoodieCombineHiveInputFormat extends HoodieCommonTestHarness {
InputFormatTestUtil.setPropsForInputFormat(jobConf, schema, tripsHiveColumnTypes); InputFormatTestUtil.setPropsForInputFormat(jobConf, schema, tripsHiveColumnTypes);
InputSplit[] splits = combineHiveInputFormat.getSplits(jobConf, 1); InputSplit[] splits = combineHiveInputFormat.getSplits(jobConf, 1);
// Since the SPLIT_SIZE is 3, we should create only 1 split with all 3 file groups // Since the SPLIT_SIZE is 3, we should create only 1 split with all 3 file groups
assertEquals(splits.length, 1); assertEquals(1, splits.length);
RecordReader<NullWritable, ArrayWritable> recordReader = RecordReader<NullWritable, ArrayWritable> recordReader =
combineHiveInputFormat.getRecordReader(splits[0], jobConf, null); combineHiveInputFormat.getRecordReader(splits[0], jobConf, null);
NullWritable nullWritable = recordReader.createKey(); NullWritable nullWritable = recordReader.createKey();

127
hudi-hadoop-mr/src/test/java/org/apache/hudi/hadoop/realtime/TestHoodieRealtimeRecordReader.java
View File

@@ -52,11 +52,9 @@ import org.apache.hadoop.mapred.FileInputFormat;
import org.apache.hadoop.mapred.FileSplit; import org.apache.hadoop.mapred.FileSplit;
import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.RecordReader; import org.apache.hadoop.mapred.RecordReader;
import org.junit.Assert; import org.junit.jupiter.api.BeforeEach;
import org.junit.Before; import org.junit.jupiter.api.Test;
import org.junit.Rule; import org.junit.jupiter.api.io.TempDir;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
import java.io.File; import java.io.File;
import java.io.IOException; import java.io.IOException;
@@ -68,29 +66,29 @@ import java.util.Set;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import static org.apache.hudi.hadoop.realtime.HoodieRealtimeRecordReader.REALTIME_SKIP_MERGE_PROP; import static org.apache.hudi.hadoop.realtime.HoodieRealtimeRecordReader.REALTIME_SKIP_MERGE_PROP;
import static org.junit.Assert.assertEquals; import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.Assert.assertFalse; import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.Assert.assertTrue; import static org.junit.jupiter.api.Assertions.assertTrue;
public class TestHoodieRealtimeRecordReader { public class TestHoodieRealtimeRecordReader {
private static final String PARTITION_COLUMN = "datestr"; private static final String PARTITION_COLUMN = "datestr";
@Rule
public TemporaryFolder basePath = new TemporaryFolder();
private JobConf jobConf; private JobConf jobConf;
private FileSystem fs; private FileSystem fs;
private Configuration hadoopConf; private Configuration hadoopConf;
@Before @BeforeEach
public void setUp() { public void setUp() {
jobConf = new JobConf(); jobConf = new JobConf();
jobConf.set(AbstractRealtimeRecordReader.MAX_DFS_STREAM_BUFFER_SIZE_PROP, String.valueOf(1024 * 1024)); jobConf.set(AbstractRealtimeRecordReader.MAX_DFS_STREAM_BUFFER_SIZE_PROP, String.valueOf(1024 * 1024));
hadoopConf = HoodieTestUtils.getDefaultHadoopConf(); hadoopConf = HoodieTestUtils.getDefaultHadoopConf();
fs = FSUtils.getFs(basePath.getRoot().getAbsolutePath(), hadoopConf); fs = FSUtils.getFs(basePath.toString(), hadoopConf);
} }
private Writer writeLogFile(File partitionDir, Schema schema, String fileId, String baseCommit, String @TempDir
newCommit, public java.nio.file.Path basePath;
private Writer writeLogFile(File partitionDir, Schema schema, String fileId, String baseCommit, String newCommit,
int numberOfRecords) throws InterruptedException, IOException { int numberOfRecords) throws InterruptedException, IOException {
return InputFormatTestUtil.writeDataBlockToLogFile(partitionDir, fs, schema, fileId, baseCommit, newCommit, return InputFormatTestUtil.writeDataBlockToLogFile(partitionDir, fs, schema, fileId, baseCommit, newCommit,
numberOfRecords, 0, numberOfRecords, 0,
@@ -125,7 +123,7 @@ public class TestHoodieRealtimeRecordReader {
private void testReader(boolean partitioned) throws Exception { private void testReader(boolean partitioned) throws Exception {
// initial commit // initial commit
Schema schema = HoodieAvroUtils.addMetadataFields(SchemaTestUtil.getEvolvedSchema()); Schema schema = HoodieAvroUtils.addMetadataFields(SchemaTestUtil.getEvolvedSchema());
HoodieTestUtils.init(hadoopConf, basePath.getRoot().getAbsolutePath(), HoodieTableType.MERGE_ON_READ); HoodieTestUtils.init(hadoopConf, basePath.toString(), HoodieTableType.MERGE_ON_READ);
String baseInstant = "100"; String baseInstant = "100";
File partitionDir = partitioned ? InputFormatTestUtil.prepareParquetTable(basePath, schema, 1, 100, baseInstant) File partitionDir = partitioned ? InputFormatTestUtil.prepareParquetTable(basePath, schema, 1, 100, baseInstant)
: InputFormatTestUtil.prepareNonPartitionedParquetTable(basePath, schema, 1, 100, baseInstant); : InputFormatTestUtil.prepareNonPartitionedParquetTable(basePath, schema, 1, 100, baseInstant);
@@ -139,7 +137,7 @@ public class TestHoodieRealtimeRecordReader {
// TODO: HUDI-154 Once Hive 2.x PR (PR-674) is merged, enable this change // TODO: HUDI-154 Once Hive 2.x PR (PR-674) is merged, enable this change
// logVersionsWithAction.add(Pair.of(HoodieTimeline.ROLLBACK_ACTION, 3)); // logVersionsWithAction.add(Pair.of(HoodieTimeline.ROLLBACK_ACTION, 3));
FileSlice fileSlice = FileSlice fileSlice =
new FileSlice(partitioned ? FSUtils.getRelativePartitionPath(new Path(basePath.getRoot().getAbsolutePath()), new FileSlice(partitioned ? FSUtils.getRelativePartitionPath(new Path(basePath.toString()),
new Path(partitionDir.getAbsolutePath())) : "default", baseInstant, "fileid0"); new Path(partitionDir.getAbsolutePath())) : "default", baseInstant, "fileid0");
logVersionsWithAction.forEach(logVersionWithAction -> { logVersionsWithAction.forEach(logVersionWithAction -> {
try { try {
@@ -163,13 +161,13 @@ public class TestHoodieRealtimeRecordReader {
} }
long size = writer.getCurrentSize(); long size = writer.getCurrentSize();
writer.close(); writer.close();
assertTrue("block - size should be > 0", size > 0); assertTrue(size > 0, "block - size should be > 0");
// create a split with baseFile (parquet file written earlier) and new log file(s) // create a split with baseFile (parquet file written earlier) and new log file(s)
fileSlice.addLogFile(writer.getLogFile()); fileSlice.addLogFile(writer.getLogFile());
HoodieRealtimeFileSplit split = new HoodieRealtimeFileSplit( HoodieRealtimeFileSplit split = new HoodieRealtimeFileSplit(
new FileSplit(new Path(partitionDir + "/fileid0_1-0-1_" + baseInstant + ".parquet"), 0, 1, jobConf), new FileSplit(new Path(partitionDir + "/fileid0_1-0-1_" + baseInstant + ".parquet"), 0, 1, jobConf),
basePath.getRoot().getPath(), fileSlice.getLogFiles().sorted(HoodieLogFile.getLogFileComparator()) basePath.toString(), fileSlice.getLogFiles().sorted(HoodieLogFile.getLogFileComparator())
.map(h -> h.getPath().toString()).collect(Collectors.toList()), .map(h -> h.getPath().toString()).collect(Collectors.toList()),
instantTime); instantTime);
@@ -210,7 +208,7 @@ public class TestHoodieRealtimeRecordReader {
public void testUnMergedReader() throws Exception { public void testUnMergedReader() throws Exception {
// initial commit // initial commit
Schema schema = HoodieAvroUtils.addMetadataFields(SchemaTestUtil.getEvolvedSchema()); Schema schema = HoodieAvroUtils.addMetadataFields(SchemaTestUtil.getEvolvedSchema());
HoodieTestUtils.init(hadoopConf, basePath.getRoot().getAbsolutePath(), HoodieTableType.MERGE_ON_READ); HoodieTestUtils.init(hadoopConf, basePath.toString(), HoodieTableType.MERGE_ON_READ);
String instantTime = "100"; String instantTime = "100";
final int numRecords = 1000; final int numRecords = 1000;
final int firstBatchLastRecordKey = numRecords - 1; final int firstBatchLastRecordKey = numRecords - 1;
@@ -227,13 +225,13 @@ public class TestHoodieRealtimeRecordReader {
numRecords, numRecords, 0); numRecords, numRecords, 0);
long size = writer.getCurrentSize(); long size = writer.getCurrentSize();
writer.close(); writer.close();
assertTrue("block - size should be > 0", size > 0); assertTrue(size > 0, "block - size should be > 0");
// create a split with baseFile (parquet file written earlier) and new log file(s) // create a split with baseFile (parquet file written earlier) and new log file(s)
String logFilePath = writer.getLogFile().getPath().toString(); String logFilePath = writer.getLogFile().getPath().toString();
HoodieRealtimeFileSplit split = new HoodieRealtimeFileSplit( HoodieRealtimeFileSplit split = new HoodieRealtimeFileSplit(
new FileSplit(new Path(partitionDir + "/fileid0_1-0-1_" + instantTime + ".parquet"), 0, 1, jobConf), new FileSplit(new Path(partitionDir + "/fileid0_1-0-1_" + instantTime + ".parquet"), 0, 1, jobConf),
basePath.getRoot().getPath(), Collections.singletonList(logFilePath), newCommitTime); basePath.toString(), Collections.singletonList(logFilePath), newCommitTime);
// create a RecordReader to be used by HoodieRealtimeRecordReader // create a RecordReader to be used by HoodieRealtimeRecordReader
RecordReader<NullWritable, ArrayWritable> reader = new MapredParquetInputFormat().getRecordReader( RecordReader<NullWritable, ArrayWritable> reader = new MapredParquetInputFormat().getRecordReader(
@@ -262,17 +260,17 @@ public class TestHoodieRealtimeRecordReader {
int gotKey = Integer.parseInt(keyStr.substring("key".length())); int gotKey = Integer.parseInt(keyStr.substring("key".length()));
if (gotCommit.equals(newCommitTime)) { if (gotCommit.equals(newCommitTime)) {
numRecordsAtCommit2++; numRecordsAtCommit2++;
Assert.assertTrue(gotKey > firstBatchLastRecordKey); assertTrue(gotKey > firstBatchLastRecordKey);
Assert.assertTrue(gotKey <= secondBatchLastRecordKey); assertTrue(gotKey <= secondBatchLastRecordKey);
assertEquals(gotKey, lastSeenKeyFromLog + 1); assertEquals(gotKey, lastSeenKeyFromLog + 1);
lastSeenKeyFromLog++; lastSeenKeyFromLog++;
} else { } else {
numRecordsAtCommit1++; numRecordsAtCommit1++;
Assert.assertTrue(gotKey >= 0); assertTrue(gotKey >= 0);
Assert.assertTrue(gotKey <= firstBatchLastRecordKey); assertTrue(gotKey <= firstBatchLastRecordKey);
} }
// Ensure unique key // Ensure unique key
Assert.assertFalse(seenKeys.contains(gotKey)); assertFalse(seenKeys.contains(gotKey));
seenKeys.add(gotKey); seenKeys.add(gotKey);
key = recordReader.createKey(); key = recordReader.createKey();
value = recordReader.createValue(); value = recordReader.createValue();
@@ -288,7 +286,7 @@ public class TestHoodieRealtimeRecordReader {
public void testReaderWithNestedAndComplexSchema() throws Exception { public void testReaderWithNestedAndComplexSchema() throws Exception {
// initial commit // initial commit
Schema schema = HoodieAvroUtils.addMetadataFields(SchemaTestUtil.getComplexEvolvedSchema()); Schema schema = HoodieAvroUtils.addMetadataFields(SchemaTestUtil.getComplexEvolvedSchema());
HoodieTestUtils.init(hadoopConf, basePath.getRoot().getAbsolutePath(), HoodieTableType.MERGE_ON_READ); HoodieTestUtils.init(hadoopConf, basePath.toString(), HoodieTableType.MERGE_ON_READ);
String instantTime = "100"; String instantTime = "100";
int numberOfRecords = 100; int numberOfRecords = 100;
int numberOfLogRecords = numberOfRecords / 2; int numberOfLogRecords = numberOfRecords / 2;
@@ -303,14 +301,14 @@ public class TestHoodieRealtimeRecordReader {
writeLogFile(partitionDir, schema, "fileid0", instantTime, newCommitTime, numberOfLogRecords); writeLogFile(partitionDir, schema, "fileid0", instantTime, newCommitTime, numberOfLogRecords);
long size = writer.getCurrentSize(); long size = writer.getCurrentSize();
writer.close(); writer.close();
assertTrue("block - size should be > 0", size > 0); assertTrue(size > 0, "block - size should be > 0");
InputFormatTestUtil.deltaCommit(basePath, newCommitTime); InputFormatTestUtil.deltaCommit(basePath, newCommitTime);
// create a split with baseFile (parquet file written earlier) and new log file(s) // create a split with baseFile (parquet file written earlier) and new log file(s)
String logFilePath = writer.getLogFile().getPath().toString(); String logFilePath = writer.getLogFile().getPath().toString();
HoodieRealtimeFileSplit split = new HoodieRealtimeFileSplit( HoodieRealtimeFileSplit split = new HoodieRealtimeFileSplit(
new FileSplit(new Path(partitionDir + "/fileid0_1-0-1_" + instantTime + ".parquet"), 0, 1, jobConf), new FileSplit(new Path(partitionDir + "/fileid0_1-0-1_" + instantTime + ".parquet"), 0, 1, jobConf),
basePath.getRoot().getPath(), Collections.singletonList(logFilePath), newCommitTime); basePath.toString(), Collections.singletonList(logFilePath), newCommitTime);
// create a RecordReader to be used by HoodieRealtimeRecordReader // create a RecordReader to be used by HoodieRealtimeRecordReader
RecordReader<NullWritable, ArrayWritable> reader = new MapredParquetInputFormat().getRecordReader( RecordReader<NullWritable, ArrayWritable> reader = new MapredParquetInputFormat().getRecordReader(
@@ -345,66 +343,69 @@ public class TestHoodieRealtimeRecordReader {
value = recordReader.createValue(); value = recordReader.createValue();
// Assert type STRING // Assert type STRING
assertEquals("test value for field: field1", values[5].toString(), "field" + currentRecordNo); assertEquals(values[5].toString(), "field" + currentRecordNo, "test value for field: field1");
assertEquals("test value for field: field2", values[6].toString(), assertEquals(values[6].toString(), "field" + currentRecordNo + recordCommitTimeSuffix,
"field" + currentRecordNo + recordCommitTimeSuffix); "test value for field: field2");
assertEquals("test value for field: name", values[7].toString(), "name" + currentRecordNo); assertEquals(values[7].toString(), "name" + currentRecordNo,
"test value for field: name");
// Assert type INT // Assert type INT
IntWritable intWritable = (IntWritable) values[8]; IntWritable intWritable = (IntWritable) values[8];
assertEquals("test value for field: favoriteIntNumber", intWritable.get(), assertEquals(intWritable.get(), currentRecordNo + recordCommitTime.hashCode(),
currentRecordNo + recordCommitTime.hashCode()); "test value for field: favoriteIntNumber");
// Assert type LONG // Assert type LONG
LongWritable longWritable = (LongWritable) values[9]; LongWritable longWritable = (LongWritable) values[9];
assertEquals("test value for field: favoriteNumber", longWritable.get(), assertEquals(longWritable.get(), currentRecordNo + recordCommitTime.hashCode(),
currentRecordNo + recordCommitTime.hashCode()); "test value for field: favoriteNumber");
// Assert type FLOAT // Assert type FLOAT
FloatWritable floatWritable = (FloatWritable) values[10]; FloatWritable floatWritable = (FloatWritable) values[10];
assertEquals("test value for field: favoriteFloatNumber", floatWritable.get(), assertEquals(floatWritable.get(), (float) ((currentRecordNo + recordCommitTime.hashCode()) / 1024.0), 0,
(float) ((currentRecordNo + recordCommitTime.hashCode()) / 1024.0), 0); "test value for field: favoriteFloatNumber");
// Assert type DOUBLE // Assert type DOUBLE
DoubleWritable doubleWritable = (DoubleWritable) values[11]; DoubleWritable doubleWritable = (DoubleWritable) values[11];
assertEquals("test value for field: favoriteDoubleNumber", doubleWritable.get(), assertEquals(doubleWritable.get(), (currentRecordNo + recordCommitTime.hashCode()) / 1024.0, 0,
(currentRecordNo + recordCommitTime.hashCode()) / 1024.0, 0); "test value for field: favoriteDoubleNumber");
// Assert type MAP // Assert type MAP
ArrayWritable mapItem = (ArrayWritable) values[12]; ArrayWritable mapItem = (ArrayWritable) values[12];
Writable mapItemValue1 = mapItem.get()[0]; Writable mapItemValue1 = mapItem.get()[0];
Writable mapItemValue2 = mapItem.get()[1]; Writable mapItemValue2 = mapItem.get()[1];
assertEquals("test value for field: tags", ((ArrayWritable) mapItemValue1).get()[0].toString(), assertEquals(((ArrayWritable) mapItemValue1).get()[0].toString(), "mapItem1",
"mapItem1"); "test value for field: tags");
assertEquals("test value for field: tags", ((ArrayWritable) mapItemValue2).get()[0].toString(), assertEquals(((ArrayWritable) mapItemValue2).get()[0].toString(), "mapItem2",
"mapItem2"); "test value for field: tags");
assertEquals("test value for field: tags", ((ArrayWritable) mapItemValue1).get().length, 2); assertEquals(((ArrayWritable) mapItemValue1).get().length, 2,
assertEquals("test value for field: tags", ((ArrayWritable) mapItemValue2).get().length, 2); "test value for field: tags");
assertEquals(((ArrayWritable) mapItemValue2).get().length, 2,
"test value for field: tags");
Writable mapItemValue1value = ((ArrayWritable) mapItemValue1).get()[1]; Writable mapItemValue1value = ((ArrayWritable) mapItemValue1).get()[1];
Writable mapItemValue2value = ((ArrayWritable) mapItemValue2).get()[1]; Writable mapItemValue2value = ((ArrayWritable) mapItemValue2).get()[1];
assertEquals("test value for field: tags[\"mapItem1\"].item1", assertEquals(((ArrayWritable) mapItemValue1value).get()[0].toString(), "item" + currentRecordNo,
((ArrayWritable) mapItemValue1value).get()[0].toString(), "item" + currentRecordNo); "test value for field: tags[\"mapItem1\"].item1");
assertEquals("test value for field: tags[\"mapItem2\"].item1", assertEquals(((ArrayWritable) mapItemValue2value).get()[0].toString(), "item2" + currentRecordNo,
((ArrayWritable) mapItemValue2value).get()[0].toString(), "item2" + currentRecordNo); "test value for field: tags[\"mapItem2\"].item1");
assertEquals("test value for field: tags[\"mapItem1\"].item2", assertEquals(((ArrayWritable) mapItemValue1value).get()[1].toString(), "item" + currentRecordNo + recordCommitTimeSuffix,
((ArrayWritable) mapItemValue1value).get()[1].toString(), "item" + currentRecordNo + recordCommitTimeSuffix); "test value for field: tags[\"mapItem1\"].item2");
assertEquals("test value for field: tags[\"mapItem2\"].item2", assertEquals(((ArrayWritable) mapItemValue2value).get()[1].toString(), "item2" + currentRecordNo + recordCommitTimeSuffix,
((ArrayWritable) mapItemValue2value).get()[1].toString(), "item2" + currentRecordNo + recordCommitTimeSuffix); "test value for field: tags[\"mapItem2\"].item2");
// Assert type RECORD // Assert type RECORD
ArrayWritable recordItem = (ArrayWritable) values[13]; ArrayWritable recordItem = (ArrayWritable) values[13];
Writable[] nestedRecord = recordItem.get(); Writable[] nestedRecord = recordItem.get();
assertFalse("test value for field: testNestedRecord.isAdmin", ((BooleanWritable) nestedRecord[0]).get()); assertFalse(((BooleanWritable) nestedRecord[0]).get(), "test value for field: testNestedRecord.isAdmin");
assertEquals("test value for field: testNestedRecord.userId", nestedRecord[1].toString(), assertEquals(nestedRecord[1].toString(), "UserId" + currentRecordNo + recordCommitTimeSuffix,
"UserId" + currentRecordNo + recordCommitTimeSuffix); "test value for field: testNestedRecord.userId");
// Assert type ARRAY // Assert type ARRAY
ArrayWritable arrayValue = (ArrayWritable) values[14]; ArrayWritable arrayValue = (ArrayWritable) values[14];
Writable[] arrayValues = arrayValue.get(); Writable[] arrayValues = arrayValue.get();
for (int i = 0; i < arrayValues.length; i++) { for (int i = 0; i < arrayValues.length; i++) {
assertEquals("test value for field: stringArray", "stringArray" + i + recordCommitTimeSuffix, assertEquals("stringArray" + i + recordCommitTimeSuffix, arrayValues[i].toString(),
arrayValues[i].toString()); "test value for field: stringArray");
} }
} }
} }
@@ -414,7 +415,7 @@ public class TestHoodieRealtimeRecordReader {
// initial commit // initial commit
List<String> logFilePaths = new ArrayList<>(); List<String> logFilePaths = new ArrayList<>();
Schema schema = HoodieAvroUtils.addMetadataFields(SchemaTestUtil.getSimpleSchema()); Schema schema = HoodieAvroUtils.addMetadataFields(SchemaTestUtil.getSimpleSchema());
HoodieTestUtils.init(hadoopConf, basePath.getRoot().getAbsolutePath(), HoodieTableType.MERGE_ON_READ); HoodieTestUtils.init(hadoopConf, basePath.toString(), HoodieTableType.MERGE_ON_READ);
String instantTime = "100"; String instantTime = "100";
int numberOfRecords = 100; int numberOfRecords = 100;
int numberOfLogRecords = numberOfRecords / 2; int numberOfLogRecords = numberOfRecords / 2;
@@ -434,7 +435,7 @@ public class TestHoodieRealtimeRecordReader {
long size = writer.getCurrentSize(); long size = writer.getCurrentSize();
logFilePaths.add(writer.getLogFile().getPath().toString()); logFilePaths.add(writer.getLogFile().getPath().toString());
writer.close(); writer.close();
assertTrue("block - size should be > 0", size > 0); assertTrue(size > 0, "block - size should be > 0");
// write rollback for the previous block in new log file version // write rollback for the previous block in new log file version
newCommitTime = "102"; newCommitTime = "102";
@@ -447,7 +448,7 @@ public class TestHoodieRealtimeRecordReader {
// create a split with baseFile (parquet file written earlier) and new log file(s) // create a split with baseFile (parquet file written earlier) and new log file(s)
HoodieRealtimeFileSplit split = new HoodieRealtimeFileSplit( HoodieRealtimeFileSplit split = new HoodieRealtimeFileSplit(
new FileSplit(new Path(partitionDir + "/fileid0_1_" + instantTime + ".parquet"), 0, 1, jobConf), new FileSplit(new Path(partitionDir + "/fileid0_1_" + instantTime + ".parquet"), 0, 1, jobConf),
basePath.getRoot().getPath(), logFilePaths, newCommitTime); basePath.toString(), logFilePaths, newCommitTime);
// create a RecordReader to be used by HoodieRealtimeRecordReader // create a RecordReader to be used by HoodieRealtimeRecordReader
RecordReader<NullWritable, ArrayWritable> reader = new MapredParquetInputFormat().getRecordReader( RecordReader<NullWritable, ArrayWritable> reader = new MapredParquetInputFormat().getRecordReader(

22
hudi-utilities/src/test/java/org/apache/hudi/utilities/TestHoodieSnapshotCopier.java
View File

@@ -18,28 +18,28 @@
package org.apache.hudi.utilities; package org.apache.hudi.utilities;
import org.apache.hudi.common.HoodieCommonTestHarness;
import org.apache.hudi.common.HoodieTestDataGenerator; import org.apache.hudi.common.HoodieTestDataGenerator;
import org.apache.hudi.common.fs.FSUtils; import org.apache.hudi.common.fs.FSUtils;
import org.apache.hudi.common.model.HoodieTestUtils; import org.apache.hudi.common.model.HoodieTestUtils;
import org.apache.hudi.common.testutils.HoodieCommonTestHarnessJunit5;
import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.Path;
import org.apache.spark.SparkConf; import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaSparkContext; import org.apache.spark.api.java.JavaSparkContext;
import org.junit.After; import org.junit.jupiter.api.AfterEach;
import org.junit.Before; import org.junit.jupiter.api.BeforeEach;
import org.junit.Test; import org.junit.jupiter.api.Test;
import java.io.File; import java.io.File;
import java.io.IOException; import java.io.IOException;
import static org.junit.Assert.assertEquals; import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.Assert.assertFalse; import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.Assert.assertTrue; import static org.junit.jupiter.api.Assertions.assertTrue;
public class TestHoodieSnapshotCopier extends HoodieCommonTestHarness { public class TestHoodieSnapshotCopier extends HoodieCommonTestHarnessJunit5 {
private static final String TEST_WRITE_TOKEN = "1-0-1"; private static final String TEST_WRITE_TOKEN = "1-0-1";
@@ -49,10 +49,10 @@ public class TestHoodieSnapshotCopier extends HoodieCommonTestHarness {
private FileSystem fs = null; private FileSystem fs = null;
private JavaSparkContext jsc = null; private JavaSparkContext jsc = null;
@Before @BeforeEach
public void init() throws IOException { public void init() throws IOException {
// Prepare directories // Prepare directories
rootPath = "file://" + folder.getRoot().getAbsolutePath(); rootPath = "file://" + tempDir.toString();
basePath = rootPath + "/" + HoodieTestUtils.RAW_TRIPS_TEST_NAME; basePath = rootPath + "/" + HoodieTestUtils.RAW_TRIPS_TEST_NAME;
outputPath = rootPath + "/output"; outputPath = rootPath + "/output";
@@ -147,7 +147,7 @@ public class TestHoodieSnapshotCopier extends HoodieCommonTestHarness {
assertTrue(fs.exists(new Path(outputPath + "/_SUCCESS"))); assertTrue(fs.exists(new Path(outputPath + "/_SUCCESS")));
} }
@After @AfterEach
public void cleanup() { public void cleanup() {
if (rootPath != null) { if (rootPath != null) {
new File(rootPath).delete(); new File(rootPath).delete();

20
hudi-utilities/src/test/java/org/apache/hudi/utilities/checkpointing/TestKafkaConnectHdfsProvider.java
View File

@@ -18,24 +18,26 @@
package org.apache.hudi.utilities.checkpointing; package org.apache.hudi.utilities.checkpointing;
import org.apache.hudi.common.HoodieCommonTestHarness;
import org.apache.hudi.common.config.TypedProperties; import org.apache.hudi.common.config.TypedProperties;
import org.apache.hudi.common.model.HoodieTestUtils; import org.apache.hudi.common.model.HoodieTestUtils;
import org.apache.hudi.common.testutils.HoodieCommonTestHarnessJunit5;
import org.apache.hudi.exception.HoodieException; import org.apache.hudi.exception.HoodieException;
import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configuration;
import org.junit.Before; import org.junit.jupiter.api.BeforeEach;
import org.junit.Test; import org.junit.jupiter.api.Test;
import java.io.File; import java.io.File;
import static org.junit.Assert.assertEquals; import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertThrows;
public class TestKafkaConnectHdfsProvider extends HoodieCommonTestHarnessJunit5 {
public class TestKafkaConnectHdfsProvider extends HoodieCommonTestHarness {
private String topicPath = null; private String topicPath = null;
private Configuration hadoopConf = null; private Configuration hadoopConf = null;
@Before @BeforeEach
public void init() { public void init() {
// Prepare directories // Prepare directories
initPath(); initPath();
@@ -71,10 +73,10 @@ public class TestKafkaConnectHdfsProvider extends HoodieCommonTestHarness {
props.put("hoodie.deltastreamer.checkpoint.provider.path", topicPath); props.put("hoodie.deltastreamer.checkpoint.provider.path", topicPath);
final InitialCheckPointProvider provider = new KafkaConnectHdfsProvider(props); final InitialCheckPointProvider provider = new KafkaConnectHdfsProvider(props);
provider.init(hadoopConf); provider.init(hadoopConf);
assertEquals(provider.getCheckpoint(), "topic1,0:300,1:200"); assertEquals("topic1,0:300,1:200", provider.getCheckpoint());
} }
@Test(expected = HoodieException.class) @Test
public void testMissingPartition() throws Exception { public void testMissingPartition() throws Exception {
topicPath = basePath + "/topic2"; topicPath = basePath + "/topic2";
new File(topicPath).mkdirs(); new File(topicPath).mkdirs();
@@ -92,6 +94,6 @@ public class TestKafkaConnectHdfsProvider extends HoodieCommonTestHarness {
props.put("hoodie.deltastreamer.checkpoint.provider.path", topicPath); props.put("hoodie.deltastreamer.checkpoint.provider.path", topicPath);
final InitialCheckPointProvider provider = new KafkaConnectHdfsProvider(props); final InitialCheckPointProvider provider = new KafkaConnectHdfsProvider(props);
provider.init(hadoopConf); provider.init(hadoopConf);
provider.getCheckpoint(); assertThrows(HoodieException.class, provider::getCheckpoint);
} }
} }