lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-995] Migrate HoodieTestUtils APIs to HoodieTestTable (#2112)

Browse Source 
Remove APIs in HoodieTestUtils

- HoodieTestUtils#createInflightCommitFiles
- HoodieTestUtils#getCommitFilePath
- HoodieTestUtils#doesCommitExist

and migrate usages to HoodieTestTable in

- hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestRollbacksCommand.java
- hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestUpgradeDowngradeCommand.java
- hudi-cli/src/test/java/org/apache/hudi/cli/integ/ITTestCommitsCommand.java
- hudi-cli/src/test/java/org/apache/hudi/cli/testutils/HoodieTestCommitMetadataGenerator.java
- hudi-client/src/test/java/org/apache/hudi/client/TestHoodieClientOnCopyOnWriteStorage.java
This commit is contained in:
Raymond Xu
2020-09-26 06:21:47 -07:00
committed by GitHub
parent ae68b2b355
commit 1be0b06ef8
10 changed files with 206 additions and 242 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestArchivedCommitsCommand.java
View File

@@ -54,7 +54,7 @@ public class TestArchivedCommitsCommand extends AbstractShellIntegrationTest {
private String tablePath;
@BeforeEach
public void init() throws IOException {
public void init() throws Exception {
initDFS();
jsc.hadoopConfiguration().addResource(dfs.getConf());
HoodieCLI.conf = dfs.getConf();
@@ -156,7 +156,7 @@ public class TestArchivedCommitsCommand extends AbstractShellIntegrationTest {
* Test for command: show archived commits.
*/
@Test
public void testShowCommits() throws IOException {
public void testShowCommits() throws Exception {
CommandResult cr = getShell().executeCommand("show archived commits");
assertTrue(cr.isSuccess());
final List<Comparable[]> rows = new ArrayList<>();

2
hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestCleansCommand.java
View File

@@ -62,7 +62,7 @@ public class TestCleansCommand extends AbstractShellIntegrationTest {
private URL propsFilePath;
@BeforeEach
public void init() throws IOException {
public void init() throws Exception {
HoodieCLI.conf = jsc.hadoopConfiguration();
String tableName = "test_table";

38
hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestCommitsCommand.java
View File

@@ -76,17 +76,19 @@ public class TestCommitsCommand extends AbstractShellIntegrationTest {
"", TimelineLayoutVersion.VERSION_1, "org.apache.hudi.common.model.HoodieAvroPayload");
}
private LinkedHashMap<String, Integer[]> generateData() {
private LinkedHashMap<String, Integer[]> generateData() throws Exception {
// generate data and metadata
LinkedHashMap<String, Integer[]> data = new LinkedHashMap<>();
data.put("102", new Integer[] {15, 10});
data.put("101", new Integer[] {20, 10});
data.put("100", new Integer[] {15, 15});
data.forEach((key, value) -> {
for (Map.Entry<String, Integer[]> entry : data.entrySet()) {
String key = entry.getKey();
Integer[] value = entry.getValue();
HoodieTestCommitMetadataGenerator.createCommitFileWithMetadata(tablePath, key, jsc.hadoopConfiguration(),
Option.of(value[0]), Option.of(value[1]));
});
}
metaClient = HoodieTableMetaClient.reload(HoodieCLI.getTableMetaClient());
assertEquals(3, metaClient.reloadActiveTimeline().getCommitsTimeline().countInstants(),
@@ -138,7 +140,7 @@ public class TestCommitsCommand extends AbstractShellIntegrationTest {
* Test case of 'commits show' command.
*/
@Test
public void testShowCommits() throws IOException {
public void testShowCommits() throws Exception {
Map<String, Integer[]> data = generateData();
CommandResult cr = getShell().executeCommand("commits show");
@@ -154,7 +156,7 @@ public class TestCommitsCommand extends AbstractShellIntegrationTest {
* Test case of 'commits showarchived' command.
*/
@Test
public void testShowArchivedCommits() throws IOException {
public void testShowArchivedCommits() throws Exception {
// Generate archive
HoodieWriteConfig cfg = HoodieWriteConfig.newBuilder().withPath(tablePath)
.withSchema(HoodieTestCommitMetadataGenerator.TRIP_EXAMPLE_SCHEMA).withParallelism(2, 2)
@@ -168,10 +170,12 @@ public class TestCommitsCommand extends AbstractShellIntegrationTest {
data.put("102", new Integer[] {25, 45});
data.put("101", new Integer[] {35, 15});
data.forEach((key, value) -> {
for (Map.Entry<String, Integer[]> entry : data.entrySet()) {
String key = entry.getKey();
Integer[] value = entry.getValue();
HoodieTestCommitMetadataGenerator.createCommitFileWithMetadata(tablePath, key, jsc.hadoopConfiguration(),
Option.of(value[0]), Option.of(value[1]));
});
}
// archive
metaClient = HoodieTableMetaClient.reload(HoodieCLI.getTableMetaClient());
@@ -198,7 +202,7 @@ public class TestCommitsCommand extends AbstractShellIntegrationTest {
* Test case of 'commit showpartitions' command.
*/
@Test
public void testShowCommitPartitions() {
public void testShowCommitPartitions() throws Exception {
Map<String, Integer[]> data = generateData();
String commitInstant = "101";
@@ -235,7 +239,7 @@ public class TestCommitsCommand extends AbstractShellIntegrationTest {
* Test case of 'commit showfiles' command.
*/
@Test
public void testShowCommitFiles() {
public void testShowCommitFiles() throws Exception {
Map<String, Integer[]> data = generateData();
String commitInstant = "101";
@@ -270,7 +274,7 @@ public class TestCommitsCommand extends AbstractShellIntegrationTest {
* Test case of 'commits compare' command.
*/
@Test
public void testCompareCommits() throws IOException {
public void testCompareCommits() throws Exception {
Map<String, Integer[]> data = generateData();
String tableName2 = "test_table2";
@@ -278,10 +282,12 @@ public class TestCommitsCommand extends AbstractShellIntegrationTest {
HoodieTestUtils.init(jsc.hadoopConfiguration(), tablePath2, getTableType());
data.remove("102");
data.forEach((key, value) -> {
for (Map.Entry<String, Integer[]> entry : data.entrySet()) {
String key = entry.getKey();
Integer[] value = entry.getValue();
HoodieTestCommitMetadataGenerator.createCommitFileWithMetadata(tablePath2, key, jsc.hadoopConfiguration(),
Option.of(value[0]), Option.of(value[1]));
});
}
CommandResult cr = getShell().executeCommand(String.format("commits compare --path %s", tablePath2));
assertTrue(cr.isSuccess());
@@ -298,7 +304,7 @@ public class TestCommitsCommand extends AbstractShellIntegrationTest {
* Test case of 'commits sync' command.
*/
@Test
public void testSyncCommits() throws IOException {
public void testSyncCommits() throws Exception {
Map<String, Integer[]> data = generateData();
String tableName2 = "test_table2";
@@ -306,10 +312,12 @@ public class TestCommitsCommand extends AbstractShellIntegrationTest {
HoodieTestUtils.init(jsc.hadoopConfiguration(), tablePath2, getTableType(), tableName2);
data.remove("102");
data.forEach((key, value) -> {
for (Map.Entry<String, Integer[]> entry : data.entrySet()) {
String key = entry.getKey();
Integer[] value = entry.getValue();
HoodieTestCommitMetadataGenerator.createCommitFileWithMetadata(tablePath2, key, jsc.hadoopConfiguration(),
Option.of(value[0]), Option.of(value[1]));
});
}
CommandResult cr = getShell().executeCommand(String.format("commits sync --path %s", tablePath2));
assertTrue(cr.isSuccess());

54
hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestRollbacksCommand.java
View File

@@ -26,12 +26,12 @@ import org.apache.hudi.cli.TableHeader;
import org.apache.hudi.cli.testutils.AbstractShellIntegrationTest;
import org.apache.hudi.client.HoodieWriteClient;
import org.apache.hudi.common.model.HoodieTableType;
import org.apache.hudi.common.table.HoodieTableMetaClient;
import org.apache.hudi.common.table.timeline.HoodieActiveTimeline;
import org.apache.hudi.common.table.timeline.HoodieInstant;
import org.apache.hudi.common.table.timeline.TimelineMetadataUtils;
import org.apache.hudi.common.table.timeline.versioning.TimelineLayoutVersion;
import org.apache.hudi.common.testutils.HoodieTestDataGenerator;
import org.apache.hudi.common.testutils.HoodieTestUtils;
import org.apache.hudi.common.testutils.HoodieTestTable;
import org.apache.hudi.common.util.collection.Pair;
import org.apache.hudi.config.HoodieIndexConfig;
import org.apache.hudi.config.HoodieWriteConfig;
@@ -41,14 +41,18 @@ import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.springframework.shell.core.CommandResult;
import java.io.File;
import java.io.IOException;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.stream.Stream;
import static org.apache.hudi.common.testutils.HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH;
import static org.apache.hudi.common.testutils.HoodieTestDataGenerator.DEFAULT_PARTITION_PATHS;
import static org.apache.hudi.common.testutils.HoodieTestDataGenerator.DEFAULT_SECOND_PARTITION_PATH;
import static org.apache.hudi.common.testutils.HoodieTestDataGenerator.DEFAULT_THIRD_PARTITION_PATH;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
@@ -59,39 +63,37 @@ import static org.junit.jupiter.api.Assertions.assertTrue;
public class TestRollbacksCommand extends AbstractShellIntegrationTest {
@BeforeEach
public void init() throws IOException {
public void init() throws Exception {
String tableName = "test_table";
String tablePath = basePath + File.separator + tableName;
String tablePath = Paths.get(basePath, tableName).toString();
new TableCommand().createTable(
tablePath, tableName, HoodieTableType.MERGE_ON_READ.name(),
"", TimelineLayoutVersion.VERSION_1, "org.apache.hudi.common.model.HoodieAvroPayload");
metaClient = HoodieTableMetaClient.reload(HoodieCLI.getTableMetaClient());
//Create some commits files and parquet files
String commitTime1 = "100";
String commitTime2 = "101";
String commitTime3 = "102";
HoodieTestDataGenerator.writePartitionMetadata(fs, HoodieTestDataGenerator.DEFAULT_PARTITION_PATHS, tablePath);
// two commit files
HoodieTestUtils.createCommitFiles(tablePath, commitTime1, commitTime2);
// one .inflight commit file
HoodieTestUtils.createInflightCommitFiles(tablePath, commitTime3);
// generate commit files for commits
for (String commitTime : Arrays.asList(commitTime1, commitTime2, commitTime3)) {
HoodieTestUtils.createDataFile(tablePath, HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH, commitTime, "file-1");
HoodieTestUtils.createDataFile(tablePath, HoodieTestDataGenerator.DEFAULT_SECOND_PARTITION_PATH, commitTime, "file-2");
HoodieTestUtils.createDataFile(tablePath, HoodieTestDataGenerator.DEFAULT_THIRD_PARTITION_PATH, commitTime, "file-3");
}
Map<String, String> partitionAndFileId = new HashMap<String, String>() {
{
put(DEFAULT_FIRST_PARTITION_PATH, "file-1");
put(DEFAULT_SECOND_PARTITION_PATH, "file-2");
put(DEFAULT_THIRD_PARTITION_PATH, "file-3");
}
};
HoodieTestTable.of(metaClient)
.withPartitionMetaFiles(DEFAULT_PARTITION_PATHS)
.addCommit("100")
.withBaseFilesInPartitions(partitionAndFileId)
.addCommit("101")
.withBaseFilesInPartitions(partitionAndFileId)
.addInflightCommit("102")
.withBaseFilesInPartitions(partitionAndFileId);
// generate two rollback
HoodieWriteConfig config = HoodieWriteConfig.newBuilder().withPath(tablePath)
.withIndexConfig(HoodieIndexConfig.newBuilder().withIndexType(HoodieIndex.IndexType.INMEMORY).build()).build();
try (HoodieWriteClient client = getHoodieWriteClient(config)) {
// Rollback inflight commit3 and commit2
client.rollback(commitTime3);
client.rollback(commitTime2);
client.rollback("102");
client.rollback("101");
}
}

14
hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestStatsCommand.java
View File

@@ -73,17 +73,19 @@ public class TestStatsCommand extends AbstractShellIntegrationTest {
* Test case for command 'stats wa'.
*/
@Test
public void testWriteAmplificationStats() {
public void testWriteAmplificationStats() throws Exception {
// generate data and metadata
Map<String, Integer[]> data = new LinkedHashMap<>();
data.put("100", new Integer[] {15, 10});
data.put("101", new Integer[] {20, 10});
data.put("102", new Integer[] {15, 15});
data.forEach((key, value) -> {
HoodieTestCommitMetadataGenerator.createCommitFileWithMetadata(tablePath, key, jsc.hadoopConfiguration(),
Option.of(value[0]), Option.of(value[1]));
});
for (Map.Entry<String, Integer[]> entry : data.entrySet()) {
String k = entry.getKey();
Integer[] v = entry.getValue();
HoodieTestCommitMetadataGenerator.createCommitFileWithMetadata(tablePath, k, jsc.hadoopConfiguration(),
Option.of(v[0]), Option.of(v[1]));
}
CommandResult cr = getShell().executeCommand("stats wa");
assertTrue(cr.isSuccess());
@@ -93,7 +95,7 @@ public class TestStatsCommand extends AbstractShellIntegrationTest {
DecimalFormat df = new DecimalFormat("#.00");
data.forEach((key, value) -> {
// there are two partitions, so need to *2
rows.add(new Comparable[]{key, value[1] * 2, value[0] * 2, df.format((float) value[0] / value[1])});
rows.add(new Comparable[] {key, value[1] * 2, value[0] * 2, df.format((float) value[0] / value[1])});
});
int totalWrite = data.values().stream().map(integers -> integers[0] * 2).mapToInt(s -> s).sum();
int totalUpdate = data.values().stream().map(integers -> integers[1] * 2).mapToInt(s -> s).sum();

60
hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestUpgradeDowngradeCommand.java
View File

@@ -27,8 +27,7 @@ import org.apache.hudi.common.table.HoodieTableMetaClient;
import org.apache.hudi.common.table.HoodieTableVersion;
import org.apache.hudi.common.table.timeline.versioning.TimelineLayoutVersion;
import org.apache.hudi.common.testutils.FileCreateUtils;
import org.apache.hudi.common.testutils.HoodieTestDataGenerator;
import org.apache.hudi.common.testutils.HoodieTestUtils;
import org.apache.hudi.common.testutils.HoodieTestTable;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
@@ -36,11 +35,14 @@ import org.apache.hadoop.fs.Path;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import java.io.File;
import java.io.IOException;
import java.util.Arrays;
import java.nio.file.Paths;
import java.util.Properties;
import static org.apache.hudi.common.testutils.HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH;
import static org.apache.hudi.common.testutils.HoodieTestDataGenerator.DEFAULT_PARTITION_PATHS;
import static org.apache.hudi.common.testutils.HoodieTestDataGenerator.DEFAULT_SECOND_PARTITION_PATH;
import static org.apache.hudi.common.testutils.HoodieTestDataGenerator.DEFAULT_THIRD_PARTITION_PATH;
import static org.junit.jupiter.api.Assertions.assertEquals;
/**
@@ -51,45 +53,31 @@ public class TestUpgradeDowngradeCommand extends AbstractShellIntegrationTest {
private String tablePath;
@BeforeEach
public void init() throws IOException {
public void init() throws Exception {
String tableName = "test_table";
tablePath = basePath + File.separator + tableName;
tablePath = Paths.get(basePath, tableName).toString();
new TableCommand().createTable(
tablePath, tableName, HoodieTableType.COPY_ON_WRITE.name(),
"", TimelineLayoutVersion.VERSION_1, "org.apache.hudi.common.model.HoodieAvroPayload");
metaClient = HoodieTableMetaClient.reload(HoodieCLI.getTableMetaClient());
//Create some commits files and parquet files
String commitTime1 = "100";
String commitTime2 = "101";
HoodieTestDataGenerator.writePartitionMetadata(fs, HoodieTestDataGenerator.DEFAULT_PARTITION_PATHS, tablePath);
// one commit file
HoodieTestUtils.createCommitFiles(tablePath, commitTime1);
// one .inflight commit file
HoodieTestUtils.createInflightCommitFiles(tablePath, commitTime2);
// generate commit files for commit 100
for (String commitTime : Arrays.asList(commitTime1)) {
HoodieTestUtils.createDataFile(tablePath, HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH, commitTime, "file-1");
HoodieTestUtils.createDataFile(tablePath, HoodieTestDataGenerator.DEFAULT_SECOND_PARTITION_PATH, commitTime, "file-2");
HoodieTestUtils.createDataFile(tablePath, HoodieTestDataGenerator.DEFAULT_THIRD_PARTITION_PATH, commitTime, "file-3");
}
// generate commit and marker files for inflight commit 101
for (String commitTime : Arrays.asList(commitTime2)) {
HoodieTestUtils.createDataFile(tablePath, HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH, commitTime, "file-1");
FileCreateUtils.createMarkerFile(tablePath, HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH, commitTime, "file-1", IOType.MERGE);
HoodieTestUtils.createDataFile(tablePath, HoodieTestDataGenerator.DEFAULT_SECOND_PARTITION_PATH, commitTime, "file-2");
FileCreateUtils.createMarkerFile(tablePath, HoodieTestDataGenerator.DEFAULT_SECOND_PARTITION_PATH, commitTime, "file-2", IOType.MERGE);
HoodieTestUtils.createDataFile(tablePath, HoodieTestDataGenerator.DEFAULT_THIRD_PARTITION_PATH, commitTime, "file-3");
FileCreateUtils.createMarkerFile(tablePath, HoodieTestDataGenerator.DEFAULT_THIRD_PARTITION_PATH, commitTime, "file-3", IOType.MERGE);
}
HoodieTestTable.of(metaClient)
.withPartitionMetaFiles(DEFAULT_PARTITION_PATHS)
.addCommit("100")
.withBaseFilesInPartition(DEFAULT_FIRST_PARTITION_PATH, "file-1")
.withBaseFilesInPartition(DEFAULT_SECOND_PARTITION_PATH, "file-2")
.withBaseFilesInPartition(DEFAULT_THIRD_PARTITION_PATH, "file-3")
.addInflightCommit("101")
.withBaseFilesInPartition(DEFAULT_FIRST_PARTITION_PATH, "file-1")
.withBaseFilesInPartition(DEFAULT_SECOND_PARTITION_PATH, "file-2")
.withBaseFilesInPartition(DEFAULT_THIRD_PARTITION_PATH, "file-3")
.withMarkerFile(DEFAULT_FIRST_PARTITION_PATH, "file-1", IOType.MERGE)
.withMarkerFile(DEFAULT_SECOND_PARTITION_PATH, "file-2", IOType.MERGE)
.withMarkerFile(DEFAULT_THIRD_PARTITION_PATH, "file-3", IOType.MERGE);
}
@Test
public void testDowngradeCommand() throws Exception {
metaClient = HoodieTableMetaClient.reload(HoodieCLI.getTableMetaClient());
// update hoodie.table.version to 1
metaClient.getTableConfig().setTableVersion(HoodieTableVersion.ONE);
try (FSDataOutputStream os = metaClient.getFs().create(new Path(metaClient.getMetaPath() + "/" + HoodieTableConfig.HOODIE_PROPERTIES_FILE), true)) {
@@ -98,7 +86,7 @@ public class TestUpgradeDowngradeCommand extends AbstractShellIntegrationTest {
metaClient = HoodieTableMetaClient.reload(HoodieCLI.getTableMetaClient());
// verify marker files for inflight commit exists
for (String partitionPath : HoodieTestDataGenerator.DEFAULT_PARTITION_PATHS) {
for (String partitionPath : DEFAULT_PARTITION_PATHS) {
assertEquals(1, FileCreateUtils.getTotalMarkerFileCount(tablePath, partitionPath, "101", IOType.MERGE));
}
@@ -112,7 +100,7 @@ public class TestUpgradeDowngradeCommand extends AbstractShellIntegrationTest {
assertTableVersionFromPropertyFile();
// verify marker files are non existant
for (String partitionPath : HoodieTestDataGenerator.DEFAULT_PARTITION_PATHS) {
for (String partitionPath : DEFAULT_PARTITION_PATHS) {
assertEquals(0, FileCreateUtils.getTotalMarkerFileCount(tablePath, partitionPath, "101", IOType.MERGE));
}
}

54
hudi-cli/src/test/java/org/apache/hudi/cli/integ/ITTestCommitsCommand.java
View File

@@ -26,17 +26,21 @@ import org.apache.hudi.common.model.HoodieTableType;
import org.apache.hudi.common.table.HoodieTableMetaClient;
import org.apache.hudi.common.table.timeline.HoodieActiveTimeline;
import org.apache.hudi.common.table.timeline.versioning.TimelineLayoutVersion;
import org.apache.hudi.common.testutils.HoodieTestDataGenerator;
import org.apache.hudi.common.testutils.HoodieTestUtils;
import org.apache.hudi.common.testutils.HoodieTestTable;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.springframework.shell.core.CommandResult;
import java.io.File;
import java.io.IOException;
import java.util.Arrays;
import java.nio.file.Paths;
import java.util.HashMap;
import java.util.Map;
import static org.apache.hudi.common.testutils.HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH;
import static org.apache.hudi.common.testutils.HoodieTestDataGenerator.DEFAULT_PARTITION_PATHS;
import static org.apache.hudi.common.testutils.HoodieTestDataGenerator.DEFAULT_SECOND_PARTITION_PATH;
import static org.apache.hudi.common.testutils.HoodieTestDataGenerator.DEFAULT_THIRD_PARTITION_PATH;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
@@ -48,43 +52,45 @@ import static org.junit.jupiter.api.Assertions.assertTrue;
*/
public class ITTestCommitsCommand extends AbstractShellIntegrationTest {
private String tablePath;
@BeforeEach
public void init() throws IOException {
String tableName = "test_table";
tablePath = basePath + File.separator + tableName;
String tableName = "test_table_" + ITTestCommitsCommand.class.getName();
String tablePath = Paths.get(basePath, tableName).toString();
HoodieCLI.conf = jsc.hadoopConfiguration();
// Create table and connect
new TableCommand().createTable(
tablePath, tableName, HoodieTableType.COPY_ON_WRITE.name(),
"", TimelineLayoutVersion.VERSION_1, "org.apache.hudi.common.model.HoodieAvroPayload");
metaClient.setBasePath(tablePath);
metaClient = HoodieTableMetaClient.reload(metaClient);
}
/**
* Test case of 'commit rollback' command.
*/
@Test
public void testRollbackCommit() throws IOException {
public void testRollbackCommit() throws Exception {
//Create some commits files and parquet files
String commitTime1 = "100";
String commitTime2 = "101";
String commitTime3 = "102";
HoodieTestDataGenerator.writePartitionMetadata(fs, HoodieTestDataGenerator.DEFAULT_PARTITION_PATHS, tablePath);
// three commit files
HoodieTestUtils.createCommitFiles(tablePath, commitTime1, commitTime2, commitTime3);
// generate commit files for commits
for (String commitTime : Arrays.asList(commitTime1, commitTime2, commitTime3)) {
HoodieTestUtils.createDataFile(tablePath, HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH, commitTime, "file-1");
HoodieTestUtils.createDataFile(tablePath, HoodieTestDataGenerator.DEFAULT_SECOND_PARTITION_PATH, commitTime, "file-2");
HoodieTestUtils.createDataFile(tablePath, HoodieTestDataGenerator.DEFAULT_THIRD_PARTITION_PATH, commitTime, "file-3");
}
Map<String, String> partitionAndFileId = new HashMap<String, String>() {
{
put(DEFAULT_FIRST_PARTITION_PATH, "file-1");
put(DEFAULT_SECOND_PARTITION_PATH, "file-2");
put(DEFAULT_THIRD_PARTITION_PATH, "file-3");
}
};
final String rollbackCommit = "102";
HoodieTestTable.of(metaClient)
.withPartitionMetaFiles(DEFAULT_PARTITION_PATHS)
.addCommit("100")
.withBaseFilesInPartitions(partitionAndFileId)
.addCommit("101")
.withBaseFilesInPartitions(partitionAndFileId)
.addCommit(rollbackCommit)
.withBaseFilesInPartitions(partitionAndFileId);
CommandResult cr = getShell().executeCommand(String.format("commit rollback --commit %s --sparkMaster %s --sparkMemory %s",
commitTime3, "local", "4G"));
rollbackCommit, "local", "4G"));
assertTrue(cr.isSuccess());
metaClient = HoodieTableMetaClient.reload(HoodieCLI.getTableMetaClient());

67
hudi-cli/src/test/java/org/apache/hudi/cli/testutils/HoodieTestCommitMetadataGenerator.java
View File

@@ -23,18 +23,14 @@ import org.apache.hudi.common.model.HoodieCommitMetadata;
import org.apache.hudi.common.model.HoodieWriteStat;
import org.apache.hudi.common.table.HoodieTableMetaClient;
import org.apache.hudi.common.table.timeline.HoodieTimeline;
import org.apache.hudi.common.testutils.FileCreateUtils;
import org.apache.hudi.common.testutils.HoodieTestDataGenerator;
import org.apache.hudi.common.testutils.HoodieTestUtils;
import org.apache.hudi.common.util.CollectionUtils;
import org.apache.hudi.common.util.Option;
import org.apache.hudi.exception.HoodieIOException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.Arrays;
import java.util.HashMap;
@@ -42,6 +38,9 @@ import java.util.List;
import java.util.Map;
import java.util.UUID;
import static org.apache.hudi.common.testutils.FileCreateUtils.baseFileName;
import static org.apache.hudi.common.util.CollectionUtils.createImmutableList;
/**
* Class to be used in tests to keep generating test inserts and updates against a corpus.
*/
@@ -62,67 +61,53 @@ public class HoodieTestCommitMetadataGenerator extends HoodieTestDataGenerator {
/**
* Create a commit file with default CommitMetadata.
*/
public static void createCommitFileWithMetadata(String basePath, String commitTime, Configuration configuration) {
public static void createCommitFileWithMetadata(String basePath, String commitTime, Configuration configuration) throws Exception {
createCommitFileWithMetadata(basePath, commitTime, configuration, Option.empty(), Option.empty());
}
public static void createCommitFileWithMetadata(String basePath, String commitTime, Configuration configuration,
Option<Integer> writes, Option<Integer> updates) {
Option<Integer> writes, Option<Integer> updates) throws Exception {
createCommitFileWithMetadata(basePath, commitTime, configuration, UUID.randomUUID().toString(),
UUID.randomUUID().toString(), writes, updates);
}
public static void createCommitFileWithMetadata(String basePath, String commitTime, Configuration configuration,
String fileId1, String fileId2, Option<Integer> writes, Option<Integer> updates) {
Arrays.asList(HoodieTimeline.makeCommitFileName(commitTime), HoodieTimeline.makeInflightCommitFileName(commitTime),
HoodieTimeline.makeRequestedCommitFileName(commitTime))
.forEach(f -> {
Path commitFile = new Path(
basePath + "/" + HoodieTableMetaClient.METAFOLDER_NAME + "/" + f);
FSDataOutputStream os = null;
try {
FileSystem fs = FSUtils.getFs(basePath, configuration);
os = fs.create(commitFile, true);
// Generate commitMetadata
HoodieCommitMetadata commitMetadata =
generateCommitMetadata(basePath, commitTime, fileId1, fileId2, writes, updates);
// Write empty commit metadata
os.writeBytes(new String(commitMetadata.toJsonString().getBytes(StandardCharsets.UTF_8)));
} catch (IOException ioe) {
throw new HoodieIOException(ioe.getMessage(), ioe);
} finally {
if (null != os) {
try {
os.close();
} catch (IOException e) {
throw new HoodieIOException(e.getMessage(), e);
}
}
}
});
String fileId1, String fileId2, Option<Integer> writes, Option<Integer> updates) throws Exception {
List<String> commitFileNames = Arrays.asList(HoodieTimeline.makeCommitFileName(commitTime), HoodieTimeline.makeInflightCommitFileName(commitTime),
HoodieTimeline.makeRequestedCommitFileName(commitTime));
for (String name : commitFileNames) {
Path commitFilePath = new Path(basePath + "/" + HoodieTableMetaClient.METAFOLDER_NAME + "/" + name);
try (FSDataOutputStream os = FSUtils.getFs(basePath, configuration).create(commitFilePath, true)) {
// Generate commitMetadata
HoodieCommitMetadata commitMetadata =
generateCommitMetadata(basePath, commitTime, fileId1, fileId2, writes, updates);
// Write empty commit metadata
os.writeBytes(new String(commitMetadata.toJsonString().getBytes(StandardCharsets.UTF_8)));
}
}
}
/**
* Generate commitMetadata in path.
*/
public static HoodieCommitMetadata generateCommitMetadata(String basePath, String commitTime) throws IOException {
public static HoodieCommitMetadata generateCommitMetadata(String basePath, String commitTime) throws Exception {
return generateCommitMetadata(basePath, commitTime, Option.empty(), Option.empty());
}
public static HoodieCommitMetadata generateCommitMetadata(String basePath, String commitTime,
Option<Integer> writes, Option<Integer> updates) throws IOException {
Option<Integer> writes, Option<Integer> updates) throws Exception {
return generateCommitMetadata(basePath, commitTime, UUID.randomUUID().toString(), UUID.randomUUID().toString(),
writes, updates);
}
public static HoodieCommitMetadata generateCommitMetadata(String basePath, String commitTime, String fileId1,
String fileId2, Option<Integer> writes, Option<Integer> updates) throws IOException {
String file1P0C0 = HoodieTestUtils.createDataFile(basePath, DEFAULT_FIRST_PARTITION_PATH, commitTime, fileId1);
String file1P1C0 = HoodieTestUtils.createDataFile(basePath, DEFAULT_SECOND_PARTITION_PATH, commitTime, fileId2);
String fileId2, Option<Integer> writes, Option<Integer> updates) throws Exception {
FileCreateUtils.createBaseFile(basePath, DEFAULT_FIRST_PARTITION_PATH, commitTime, fileId1);
FileCreateUtils.createBaseFile(basePath, DEFAULT_SECOND_PARTITION_PATH, commitTime, fileId2);
return generateCommitMetadata(new HashMap<String, List<String>>() {
{
put(DEFAULT_FIRST_PARTITION_PATH, CollectionUtils.createImmutableList(file1P0C0));
put(DEFAULT_SECOND_PARTITION_PATH, CollectionUtils.createImmutableList(file1P1C0));
put(DEFAULT_FIRST_PARTITION_PATH, createImmutableList(baseFileName(DEFAULT_FIRST_PARTITION_PATH, fileId1)));
put(DEFAULT_SECOND_PARTITION_PATH, createImmutableList(baseFileName(DEFAULT_SECOND_PARTITION_PATH, fileId2)));
}
}, writes, updates);
}

126
hudi-client/src/test/java/org/apache/hudi/client/TestHoodieClientOnCopyOnWriteStorage.java
View File

@@ -33,7 +33,7 @@ import org.apache.hudi.common.table.timeline.HoodieTimeline;
import org.apache.hudi.common.table.timeline.versioning.TimelineLayoutVersion;
import org.apache.hudi.common.table.view.TableFileSystemView.BaseFileOnlyView;
import org.apache.hudi.common.testutils.HoodieTestDataGenerator;
import org.apache.hudi.common.testutils.HoodieTestUtils;
import org.apache.hudi.common.testutils.HoodieTestTable;
import org.apache.hudi.common.testutils.RawTripTestPayload;
import org.apache.hudi.common.util.FileIOUtils;
import org.apache.hudi.common.util.Option;
@@ -53,20 +53,22 @@ import org.apache.hudi.table.MarkerFiles;
import org.apache.hudi.table.action.commit.WriteHelper;
import org.apache.hudi.testutils.HoodieClientTestBase;
import org.apache.hudi.testutils.HoodieClientTestUtils;
import org.apache.hudi.testutils.HoodieWriteableTestTable;
import org.apache.avro.generic.GenericRecord;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.Path;
import org.apache.log4j.LogManager;
import org.apache.log4j.Logger;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.sql.Dataset;
import org.apache.spark.sql.Row;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.EnumSource;
import org.junit.jupiter.params.provider.ValueSource;
import java.io.FileInputStream;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
@@ -102,6 +104,12 @@ import static org.mockito.Mockito.when;
public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
private static final Logger LOG = LogManager.getLogger(TestHoodieClientOnCopyOnWriteStorage.class);
private HoodieTestTable testTable;
@BeforeEach
public void setUpTestTable() {
testTable = HoodieWriteableTestTable.of(metaClient);
}
/**
* Test Auto Commit behavior for HoodieWriteClient insert API.
@@ -170,10 +178,10 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
JavaRDD<WriteStatus> result = insertFirstBatch(cfg, client, newCommitTime, prevCommitTime, numRecords, writeFn,
isPrepped, false, numRecords);
assertFalse(HoodieTestUtils.doesCommitExist(basePath, newCommitTime),
assertFalse(testTable.commitExists(newCommitTime),
"If Autocommit is false, then commit should not be made automatically");
assertTrue(client.commit(newCommitTime, result), "Commit should succeed");
assertTrue(HoodieTestUtils.doesCommitExist(basePath, newCommitTime),
assertTrue(testTable.commitExists(newCommitTime),
"After explicit commit, commit file should be created");
}
}
@@ -985,7 +993,7 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
JavaRDD<WriteStatus> result = client.bulkInsert(writeRecords, instantTime);
assertTrue(client.commit(instantTime, result), "Commit should succeed");
assertTrue(HoodieTestUtils.doesCommitExist(basePath, instantTime),
assertTrue(testTable.commitExists(instantTime),
"After explicit commit, commit file should be created");
// Get parquet file paths from commit metadata
@@ -998,16 +1006,14 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
Collection<String> commitPathNames = commitMetadata.getFileIdAndFullPaths(basePath).values();
// Read from commit file
String filename = HoodieTestUtils.getCommitFilePath(basePath, instantTime);
FileInputStream inputStream = new FileInputStream(filename);
String everything = FileIOUtils.readAsUTFString(inputStream);
HoodieCommitMetadata metadata = HoodieCommitMetadata.fromJsonString(everything, HoodieCommitMetadata.class);
HashMap<String, String> paths = metadata.getFileIdAndFullPaths(basePath);
inputStream.close();
// Compare values in both to make sure they are equal.
for (String pathName : paths.values()) {
assertTrue(commitPathNames.contains(pathName));
try (FSDataInputStream inputStream = fs.open(testTable.getCommitFilePath(instantTime))) {
String everything = FileIOUtils.readAsUTFString(inputStream);
HoodieCommitMetadata metadata = HoodieCommitMetadata.fromJsonString(everything, HoodieCommitMetadata.class);
HashMap<String, String> paths = metadata.getFileIdAndFullPaths(basePath);
// Compare values in both to make sure they are equal.
for (String pathName : paths.values()) {
assertTrue(commitPathNames.contains(pathName));
}
}
}
}
@@ -1017,65 +1023,61 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
*/
@Test
public void testMetadataStatsOnCommit() throws Exception {
HoodieWriteConfig cfg = getConfigBuilder().withAutoCommit(false).build();
HoodieWriteClient client = getHoodieWriteClient(cfg);
HoodieTableMetaClient metaClient = new HoodieTableMetaClient(hadoopConf, basePath);
String instantTime = "000";
client.startCommitWithTime(instantTime);
String instantTime0 = "000";
client.startCommitWithTime(instantTime0);
List<HoodieRecord> records = dataGen.generateInserts(instantTime, 200);
JavaRDD<HoodieRecord> writeRecords = jsc.parallelize(records, 1);
List<HoodieRecord> records0 = dataGen.generateInserts(instantTime0, 200);
JavaRDD<HoodieRecord> writeRecords0 = jsc.parallelize(records0, 1);
JavaRDD<WriteStatus> result0 = client.bulkInsert(writeRecords0, instantTime0);
JavaRDD<WriteStatus> result = client.bulkInsert(writeRecords, instantTime);
assertTrue(client.commit(instantTime, result), "Commit should succeed");
assertTrue(HoodieTestUtils.doesCommitExist(basePath, instantTime),
assertTrue(client.commit(instantTime0, result0), "Commit should succeed");
assertTrue(testTable.commitExists(instantTime0),
"After explicit commit, commit file should be created");
// Read from commit file
String filename = HoodieTestUtils.getCommitFilePath(basePath, instantTime);
FileInputStream inputStream = new FileInputStream(filename);
String everything = FileIOUtils.readAsUTFString(inputStream);
HoodieCommitMetadata metadata =
HoodieCommitMetadata.fromJsonString(everything.toString(), HoodieCommitMetadata.class);
int inserts = 0;
for (Map.Entry<String, List<HoodieWriteStat>> pstat : metadata.getPartitionToWriteStats().entrySet()) {
for (HoodieWriteStat stat : pstat.getValue()) {
inserts += stat.getNumInserts();
try (FSDataInputStream inputStream = fs.open(testTable.getCommitFilePath(instantTime0))) {
String everything = FileIOUtils.readAsUTFString(inputStream);
HoodieCommitMetadata metadata =
HoodieCommitMetadata.fromJsonString(everything, HoodieCommitMetadata.class);
int inserts = 0;
for (Map.Entry<String, List<HoodieWriteStat>> pstat : metadata.getPartitionToWriteStats().entrySet()) {
for (HoodieWriteStat stat : pstat.getValue()) {
inserts += stat.getNumInserts();
}
}
assertEquals(200, inserts);
}
assertEquals(200, inserts);
// Update + Inserts such that they just expand file1
instantTime = "001";
client.startCommitWithTime(instantTime);
String instantTime1 = "001";
client.startCommitWithTime(instantTime1);
records = dataGen.generateUpdates(instantTime, records);
writeRecords = jsc.parallelize(records, 1);
result = client.upsert(writeRecords, instantTime);
List<HoodieRecord> records1 = dataGen.generateUpdates(instantTime1, records0);
JavaRDD<HoodieRecord> writeRecords1 = jsc.parallelize(records1, 1);
JavaRDD<WriteStatus> result1 = client.upsert(writeRecords1, instantTime1);
assertTrue(client.commit(instantTime, result), "Commit should succeed");
assertTrue(HoodieTestUtils.doesCommitExist(basePath, instantTime),
assertTrue(client.commit(instantTime1, result1), "Commit should succeed");
assertTrue(testTable.commitExists(instantTime1),
"After explicit commit, commit file should be created");
// Read from commit file
filename = HoodieTestUtils.getCommitFilePath(basePath, instantTime);
inputStream = new FileInputStream(filename);
everything = FileIOUtils.readAsUTFString(inputStream);
metadata = HoodieCommitMetadata.fromJsonString(everything.toString(), HoodieCommitMetadata.class);
inserts = 0;
int upserts = 0;
for (Map.Entry<String, List<HoodieWriteStat>> pstat : metadata.getPartitionToWriteStats().entrySet()) {
for (HoodieWriteStat stat : pstat.getValue()) {
inserts += stat.getNumInserts();
upserts += stat.getNumUpdateWrites();
try (FSDataInputStream inputStream = fs.open(testTable.getCommitFilePath(instantTime1))) {
String everything = FileIOUtils.readAsUTFString(inputStream);
HoodieCommitMetadata metadata = HoodieCommitMetadata.fromJsonString(everything, HoodieCommitMetadata.class);
int inserts = 0;
int upserts = 0;
for (Map.Entry<String, List<HoodieWriteStat>> pstat : metadata.getPartitionToWriteStats().entrySet()) {
for (HoodieWriteStat stat : pstat.getValue()) {
inserts += stat.getNumInserts();
upserts += stat.getNumUpdateWrites();
}
}
assertEquals(0, inserts);
assertEquals(200, upserts);
}
assertEquals(0, inserts);
assertEquals(200, upserts);
}
/**
@@ -1095,13 +1097,13 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
metaClient.getFs().delete(result.getKey(), false);
if (!enableOptimisticConsistencyGuard) {
assertTrue(client.commit(instantTime, result.getRight()), "Commit should succeed");
assertTrue(HoodieTestUtils.doesCommitExist(basePath, instantTime),
assertTrue(testTable.commitExists(instantTime),
"After explicit commit, commit file should be created");
// Marker directory must be removed
assertFalse(metaClient.getFs().exists(new Path(metaClient.getMarkerFolderPath(instantTime))));
} else {
// with optimistic, first client.commit should have succeeded.
assertTrue(HoodieTestUtils.doesCommitExist(basePath, instantTime),
assertTrue(testTable.commitExists(instantTime),
"After explicit commit, commit file should be created");
// Marker directory must be removed
assertFalse(metaClient.getFs().exists(new Path(metaClient.getMarkerFolderPath(instantTime))));
@@ -1124,13 +1126,13 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
if (!enableOptimisticConsistencyGuard) {
// Rollback of this commit should succeed with FailSafeCG
client.rollback(instantTime);
assertFalse(HoodieTestUtils.doesCommitExist(basePath, instantTime),
assertFalse(testTable.commitExists(instantTime),
"After explicit rollback, commit file should not be present");
// Marker directory must be removed after rollback
assertFalse(metaClient.getFs().exists(new Path(metaClient.getMarkerFolderPath(instantTime))));
} else {
// if optimistic CG is enabled, commit should have succeeded.
assertTrue(HoodieTestUtils.doesCommitExist(basePath, instantTime),
assertTrue(testTable.commitExists(instantTime),
"With optimistic CG, first commit should succeed. commit file should be present");
// Marker directory must be removed after rollback
assertFalse(metaClient.getFs().exists(new Path(metaClient.getMarkerFolderPath(instantTime))));
@@ -1145,7 +1147,7 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
} else {
// rollback of a completed commit should succeed if using list based rollback
client.rollback(instantTime);
assertFalse(HoodieTestUtils.doesCommitExist(basePath, instantTime),
assertFalse(testTable.commitExists(instantTime),
"After explicit rollback, commit file should not be present");
}
}
@@ -1216,7 +1218,7 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
JavaRDD<HoodieRecord> writeRecords = jsc.parallelize(dataGen.generateInserts(firstInstantTime, numRecords), 1);
JavaRDD<WriteStatus> result = client.bulkInsert(writeRecords, firstInstantTime);
assertTrue(client.commit(firstInstantTime, result), "Commit should succeed");
assertTrue(HoodieTestUtils.doesCommitExist(basePath, firstInstantTime),
assertTrue(testTable.commitExists(firstInstantTime),
"After explicit commit, commit file should be created");
// Check the entire dataset has all records still
@@ -1235,7 +1237,7 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
JavaRDD<WriteStatus> inserts = client.bulkInsert(insertRecords, nextInstantTime);
JavaRDD<WriteStatus> upserts = client.upsert(updateRecords, nextInstantTime);
assertTrue(client.commit(nextInstantTime, inserts.union(upserts)), "Commit should succeed");
assertTrue(HoodieTestUtils.doesCommitExist(basePath, firstInstantTime),
assertTrue(testTable.commitExists(firstInstantTime),
"After explicit commit, commit file should be created");
int totalRecords = 2 * numRecords;
assertEquals(totalRecords, HoodieClientTestUtils.read(jsc, basePath, sqlContext, fs, fullPartitionPaths).count(),

29
hudi-common/src/test/java/org/apache/hudi/common/testutils/HoodieTestUtils.java
View File

@@ -172,19 +172,6 @@ public class HoodieTestUtils {
}
}
/**
* @deprecated Use {@link HoodieTestTable} instead.
*/
public static void createInflightCommitFiles(String basePath, String... instantTimes) throws IOException {
for (String instantTime : instantTimes) {
new File(basePath + "/" + HoodieTableMetaClient.METAFOLDER_NAME + "/"
+ HoodieTimeline.makeRequestedCommitFileName(instantTime)).createNewFile();
new File(basePath + "/" + HoodieTableMetaClient.METAFOLDER_NAME + "/" + HoodieTimeline.makeInflightCommitFileName(
instantTime)).createNewFile();
}
}
public static void createPendingCleanFiles(HoodieTableMetaClient metaClient, String... instantTimes) {
for (String instantTime : instantTimes) {
Arrays.asList(HoodieTimeline.makeRequestedCleanerFileName(instantTime),
@@ -257,22 +244,6 @@ public class HoodieTestUtils {
TimelineMetadataUtils.serializeCompactionPlan(plan));
}
/**
* @deprecated Use {@link HoodieTestTable} instead.
*/
public static String getCommitFilePath(String basePath, String instantTime) {
return basePath + "/" + HoodieTableMetaClient.METAFOLDER_NAME + "/" + instantTime + HoodieTimeline.COMMIT_EXTENSION;
}
/**
* @deprecated Use {@link HoodieTestTable} instead.
*/
public static boolean doesCommitExist(String basePath, String instantTime) {
return new File(
basePath + "/" + HoodieTableMetaClient.METAFOLDER_NAME + "/" + instantTime + HoodieTimeline.COMMIT_EXTENSION)
.exists();
}
public static void createCleanFiles(HoodieTableMetaClient metaClient, String basePath,
String instantTime, Configuration configuration)
throws IOException {