[HUDI-995] Migrate HoodieTestUtils APIs to HoodieTestTable (#2112)

Remove APIs in HoodieTestUtils - HoodieTestUtils#createInflightCommitFiles - HoodieTestUtils#getCommitFilePath - HoodieTestUtils#doesCommitExist and migrate usages to HoodieTestTable in - hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestRollbacksCommand.java - hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestUpgradeDowngradeCommand.java - hudi-cli/src/test/java/org/apache/hudi/cli/integ/ITTestCommitsCommand.java - hudi-cli/src/test/java/org/apache/hudi/cli/testutils/HoodieTestCommitMetadataGenerator.java - hudi-client/src/test/java/org/apache/hudi/client/TestHoodieClientOnCopyOnWriteStorage.java
2020-09-26 06:21:47 -07:00
parent ae68b2b355
commit 1be0b06ef8
10 changed files with 206 additions and 242 deletions
--- a/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestArchivedCommitsCommand.java
+++ b/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestArchivedCommitsCommand.java
@@ -54,7 +54,7 @@ public class TestArchivedCommitsCommand extends AbstractShellIntegrationTest {
  private String tablePath;

  @BeforeEach
-  public void init() throws IOException {
+  public void init() throws Exception {
    initDFS();
    jsc.hadoopConfiguration().addResource(dfs.getConf());
    HoodieCLI.conf = dfs.getConf();
@@ -156,7 +156,7 @@ public class TestArchivedCommitsCommand extends AbstractShellIntegrationTest {
   * Test for command: show archived commits.
   */
  @Test
-  public void testShowCommits() throws IOException {
+  public void testShowCommits() throws Exception {
    CommandResult cr = getShell().executeCommand("show archived commits");
    assertTrue(cr.isSuccess());
    final List<Comparable[]> rows = new ArrayList<>();
--- a/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestCleansCommand.java
+++ b/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestCleansCommand.java
@@ -62,7 +62,7 @@ public class TestCleansCommand extends AbstractShellIntegrationTest {
  private URL propsFilePath;

  @BeforeEach
-  public void init() throws IOException {
+  public void init() throws Exception {
    HoodieCLI.conf = jsc.hadoopConfiguration();

    String tableName = "test_table";
--- a/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestCommitsCommand.java
+++ b/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestCommitsCommand.java
@@ -76,17 +76,19 @@ public class TestCommitsCommand extends AbstractShellIntegrationTest {
        "", TimelineLayoutVersion.VERSION_1, "org.apache.hudi.common.model.HoodieAvroPayload");
  }

-  private LinkedHashMap<String, Integer[]> generateData() {
+  private LinkedHashMap<String, Integer[]> generateData() throws Exception {
    // generate data and metadata
    LinkedHashMap<String, Integer[]> data = new LinkedHashMap<>();
    data.put("102", new Integer[] {15, 10});
    data.put("101", new Integer[] {20, 10});
    data.put("100", new Integer[] {15, 15});

-    data.forEach((key, value) -> {
+    for (Map.Entry<String, Integer[]> entry : data.entrySet()) {
+      String key = entry.getKey();
+      Integer[] value = entry.getValue();
      HoodieTestCommitMetadataGenerator.createCommitFileWithMetadata(tablePath, key, jsc.hadoopConfiguration(),
          Option.of(value[0]), Option.of(value[1]));
-    });
+    }

    metaClient = HoodieTableMetaClient.reload(HoodieCLI.getTableMetaClient());
    assertEquals(3, metaClient.reloadActiveTimeline().getCommitsTimeline().countInstants(),
@@ -138,7 +140,7 @@ public class TestCommitsCommand extends AbstractShellIntegrationTest {
   * Test case of 'commits show' command.
   */
  @Test
-  public void testShowCommits() throws IOException {
+  public void testShowCommits() throws Exception {
    Map<String, Integer[]> data = generateData();

    CommandResult cr = getShell().executeCommand("commits show");
@@ -154,7 +156,7 @@ public class TestCommitsCommand extends AbstractShellIntegrationTest {
   * Test case of 'commits showarchived' command.
   */
  @Test
-  public void testShowArchivedCommits() throws IOException {
+  public void testShowArchivedCommits() throws Exception {
    // Generate archive
    HoodieWriteConfig cfg = HoodieWriteConfig.newBuilder().withPath(tablePath)
        .withSchema(HoodieTestCommitMetadataGenerator.TRIP_EXAMPLE_SCHEMA).withParallelism(2, 2)
@@ -168,10 +170,12 @@ public class TestCommitsCommand extends AbstractShellIntegrationTest {
    data.put("102", new Integer[] {25, 45});
    data.put("101", new Integer[] {35, 15});

-    data.forEach((key, value) -> {
+    for (Map.Entry<String, Integer[]> entry : data.entrySet()) {
+      String key = entry.getKey();
+      Integer[] value = entry.getValue();
      HoodieTestCommitMetadataGenerator.createCommitFileWithMetadata(tablePath, key, jsc.hadoopConfiguration(),
          Option.of(value[0]), Option.of(value[1]));
-    });
+    }

    // archive
    metaClient = HoodieTableMetaClient.reload(HoodieCLI.getTableMetaClient());
@@ -198,7 +202,7 @@ public class TestCommitsCommand extends AbstractShellIntegrationTest {
   * Test case of 'commit showpartitions' command.
   */
  @Test
-  public void testShowCommitPartitions() {
+  public void testShowCommitPartitions() throws Exception {
    Map<String, Integer[]> data = generateData();

    String commitInstant = "101";
@@ -235,7 +239,7 @@ public class TestCommitsCommand extends AbstractShellIntegrationTest {
   * Test case of 'commit showfiles' command.
   */
  @Test
-  public void testShowCommitFiles() {
+  public void testShowCommitFiles() throws Exception {
    Map<String, Integer[]> data = generateData();

    String commitInstant = "101";
@@ -270,7 +274,7 @@ public class TestCommitsCommand extends AbstractShellIntegrationTest {
   * Test case of 'commits compare' command.
   */
  @Test
-  public void testCompareCommits() throws IOException {
+  public void testCompareCommits() throws Exception {
    Map<String, Integer[]> data = generateData();

    String tableName2 = "test_table2";
@@ -278,10 +282,12 @@ public class TestCommitsCommand extends AbstractShellIntegrationTest {
    HoodieTestUtils.init(jsc.hadoopConfiguration(), tablePath2, getTableType());

    data.remove("102");
-    data.forEach((key, value) -> {
+    for (Map.Entry<String, Integer[]> entry : data.entrySet()) {
+      String key = entry.getKey();
+      Integer[] value = entry.getValue();
      HoodieTestCommitMetadataGenerator.createCommitFileWithMetadata(tablePath2, key, jsc.hadoopConfiguration(),
          Option.of(value[0]), Option.of(value[1]));
-    });
+    }

    CommandResult cr = getShell().executeCommand(String.format("commits compare --path %s", tablePath2));
    assertTrue(cr.isSuccess());
@@ -298,7 +304,7 @@ public class TestCommitsCommand extends AbstractShellIntegrationTest {
   * Test case of 'commits sync' command.
   */
  @Test
-  public void testSyncCommits() throws IOException {
+  public void testSyncCommits() throws Exception {
    Map<String, Integer[]> data = generateData();

    String tableName2 = "test_table2";
@@ -306,10 +312,12 @@ public class TestCommitsCommand extends AbstractShellIntegrationTest {
    HoodieTestUtils.init(jsc.hadoopConfiguration(), tablePath2, getTableType(), tableName2);

    data.remove("102");
-    data.forEach((key, value) -> {
+    for (Map.Entry<String, Integer[]> entry : data.entrySet()) {
+      String key = entry.getKey();
+      Integer[] value = entry.getValue();
      HoodieTestCommitMetadataGenerator.createCommitFileWithMetadata(tablePath2, key, jsc.hadoopConfiguration(),
          Option.of(value[0]), Option.of(value[1]));
-    });
+    }

    CommandResult cr = getShell().executeCommand(String.format("commits sync --path %s", tablePath2));
    assertTrue(cr.isSuccess());
--- a/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestRollbacksCommand.java
+++ b/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestRollbacksCommand.java
@@ -26,12 +26,12 @@ import org.apache.hudi.cli.TableHeader;
 import org.apache.hudi.cli.testutils.AbstractShellIntegrationTest;
 import org.apache.hudi.client.HoodieWriteClient;
 import org.apache.hudi.common.model.HoodieTableType;
+import org.apache.hudi.common.table.HoodieTableMetaClient;
 import org.apache.hudi.common.table.timeline.HoodieActiveTimeline;
 import org.apache.hudi.common.table.timeline.HoodieInstant;
 import org.apache.hudi.common.table.timeline.TimelineMetadataUtils;
 import org.apache.hudi.common.table.timeline.versioning.TimelineLayoutVersion;
-import org.apache.hudi.common.testutils.HoodieTestDataGenerator;
-import org.apache.hudi.common.testutils.HoodieTestUtils;
+import org.apache.hudi.common.testutils.HoodieTestTable;
 import org.apache.hudi.common.util.collection.Pair;
 import org.apache.hudi.config.HoodieIndexConfig;
 import org.apache.hudi.config.HoodieWriteConfig;
@@ -41,14 +41,18 @@ import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 import org.springframework.shell.core.CommandResult;

-import java.io.File;
 import java.io.IOException;
+import java.nio.file.Paths;
 import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 import java.util.stream.Stream;

+import static org.apache.hudi.common.testutils.HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH;
+import static org.apache.hudi.common.testutils.HoodieTestDataGenerator.DEFAULT_PARTITION_PATHS;
+import static org.apache.hudi.common.testutils.HoodieTestDataGenerator.DEFAULT_SECOND_PARTITION_PATH;
+import static org.apache.hudi.common.testutils.HoodieTestDataGenerator.DEFAULT_THIRD_PARTITION_PATH;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertNotNull;
 import static org.junit.jupiter.api.Assertions.assertTrue;
@@ -59,39 +63,37 @@ import static org.junit.jupiter.api.Assertions.assertTrue;
 public class TestRollbacksCommand extends AbstractShellIntegrationTest {

  @BeforeEach
-  public void init() throws IOException {
+  public void init() throws Exception {
    String tableName = "test_table";
-    String tablePath = basePath + File.separator + tableName;
+    String tablePath = Paths.get(basePath, tableName).toString();
    new TableCommand().createTable(
        tablePath, tableName, HoodieTableType.MERGE_ON_READ.name(),
        "", TimelineLayoutVersion.VERSION_1, "org.apache.hudi.common.model.HoodieAvroPayload");
-
+    metaClient = HoodieTableMetaClient.reload(HoodieCLI.getTableMetaClient());
    //Create some commits files and parquet files
-    String commitTime1 = "100";
-    String commitTime2 = "101";
-    String commitTime3 = "102";
-    HoodieTestDataGenerator.writePartitionMetadata(fs, HoodieTestDataGenerator.DEFAULT_PARTITION_PATHS, tablePath);
-
-    // two commit files
-    HoodieTestUtils.createCommitFiles(tablePath, commitTime1, commitTime2);
-    // one .inflight commit file
-    HoodieTestUtils.createInflightCommitFiles(tablePath, commitTime3);
-
-    // generate commit files for commits
-    for (String commitTime : Arrays.asList(commitTime1, commitTime2, commitTime3)) {
-      HoodieTestUtils.createDataFile(tablePath, HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH, commitTime, "file-1");
-      HoodieTestUtils.createDataFile(tablePath, HoodieTestDataGenerator.DEFAULT_SECOND_PARTITION_PATH, commitTime, "file-2");
-      HoodieTestUtils.createDataFile(tablePath, HoodieTestDataGenerator.DEFAULT_THIRD_PARTITION_PATH, commitTime, "file-3");
-    }
-
+    Map<String, String> partitionAndFileId = new HashMap<String, String>() {
+      {
+        put(DEFAULT_FIRST_PARTITION_PATH, "file-1");
+        put(DEFAULT_SECOND_PARTITION_PATH, "file-2");
+        put(DEFAULT_THIRD_PARTITION_PATH, "file-3");
+      }
+    };
+    HoodieTestTable.of(metaClient)
+        .withPartitionMetaFiles(DEFAULT_PARTITION_PATHS)
+        .addCommit("100")
+        .withBaseFilesInPartitions(partitionAndFileId)
+        .addCommit("101")
+        .withBaseFilesInPartitions(partitionAndFileId)
+        .addInflightCommit("102")
+        .withBaseFilesInPartitions(partitionAndFileId);
    // generate two rollback
    HoodieWriteConfig config = HoodieWriteConfig.newBuilder().withPath(tablePath)
        .withIndexConfig(HoodieIndexConfig.newBuilder().withIndexType(HoodieIndex.IndexType.INMEMORY).build()).build();

    try (HoodieWriteClient client = getHoodieWriteClient(config)) {
      // Rollback inflight commit3 and commit2
-      client.rollback(commitTime3);
-      client.rollback(commitTime2);
+      client.rollback("102");
+      client.rollback("101");
    }
  }

--- a/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestStatsCommand.java
+++ b/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestStatsCommand.java
@@ -73,17 +73,19 @@ public class TestStatsCommand extends AbstractShellIntegrationTest {
   * Test case for command 'stats wa'.
   */
  @Test
-  public void testWriteAmplificationStats() {
+  public void testWriteAmplificationStats() throws Exception {
    // generate data and metadata
    Map<String, Integer[]> data = new LinkedHashMap<>();
    data.put("100", new Integer[] {15, 10});
    data.put("101", new Integer[] {20, 10});
    data.put("102", new Integer[] {15, 15});

-    data.forEach((key, value) -> {
-      HoodieTestCommitMetadataGenerator.createCommitFileWithMetadata(tablePath, key, jsc.hadoopConfiguration(),
-          Option.of(value[0]), Option.of(value[1]));
-    });
+    for (Map.Entry<String, Integer[]> entry : data.entrySet()) {
+      String k = entry.getKey();
+      Integer[] v = entry.getValue();
+      HoodieTestCommitMetadataGenerator.createCommitFileWithMetadata(tablePath, k, jsc.hadoopConfiguration(),
+          Option.of(v[0]), Option.of(v[1]));
+    }

    CommandResult cr = getShell().executeCommand("stats wa");
    assertTrue(cr.isSuccess());
@@ -93,7 +95,7 @@ public class TestStatsCommand extends AbstractShellIntegrationTest {
    DecimalFormat df = new DecimalFormat("#.00");
    data.forEach((key, value) -> {
      // there are two partitions, so need to *2
-      rows.add(new Comparable[]{key, value[1] * 2, value[0] * 2, df.format((float) value[0] / value[1])});
+      rows.add(new Comparable[] {key, value[1] * 2, value[0] * 2, df.format((float) value[0] / value[1])});
    });
    int totalWrite = data.values().stream().map(integers -> integers[0] * 2).mapToInt(s -> s).sum();
    int totalUpdate = data.values().stream().map(integers -> integers[1] * 2).mapToInt(s -> s).sum();
--- a/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestUpgradeDowngradeCommand.java
+++ b/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestUpgradeDowngradeCommand.java
@@ -27,8 +27,7 @@ import org.apache.hudi.common.table.HoodieTableMetaClient;
 import org.apache.hudi.common.table.HoodieTableVersion;
 import org.apache.hudi.common.table.timeline.versioning.TimelineLayoutVersion;
 import org.apache.hudi.common.testutils.FileCreateUtils;
-import org.apache.hudi.common.testutils.HoodieTestDataGenerator;
-import org.apache.hudi.common.testutils.HoodieTestUtils;
+import org.apache.hudi.common.testutils.HoodieTestTable;

 import org.apache.hadoop.fs.FSDataInputStream;
 import org.apache.hadoop.fs.FSDataOutputStream;
@@ -36,11 +35,14 @@ import org.apache.hadoop.fs.Path;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;

-import java.io.File;
 import java.io.IOException;
-import java.util.Arrays;
+import java.nio.file.Paths;
 import java.util.Properties;

+import static org.apache.hudi.common.testutils.HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH;
+import static org.apache.hudi.common.testutils.HoodieTestDataGenerator.DEFAULT_PARTITION_PATHS;
+import static org.apache.hudi.common.testutils.HoodieTestDataGenerator.DEFAULT_SECOND_PARTITION_PATH;
+import static org.apache.hudi.common.testutils.HoodieTestDataGenerator.DEFAULT_THIRD_PARTITION_PATH;
 import static org.junit.jupiter.api.Assertions.assertEquals;

 /**
@@ -51,45 +53,31 @@ public class TestUpgradeDowngradeCommand extends AbstractShellIntegrationTest {
  private String tablePath;

  @BeforeEach
-  public void init() throws IOException {
+  public void init() throws Exception {
    String tableName = "test_table";
-    tablePath = basePath + File.separator + tableName;
+    tablePath = Paths.get(basePath, tableName).toString();
    new TableCommand().createTable(
        tablePath, tableName, HoodieTableType.COPY_ON_WRITE.name(),
        "", TimelineLayoutVersion.VERSION_1, "org.apache.hudi.common.model.HoodieAvroPayload");
-
+    metaClient = HoodieTableMetaClient.reload(HoodieCLI.getTableMetaClient());
    //Create some commits files and parquet files
-    String commitTime1 = "100";
-    String commitTime2 = "101";
-    HoodieTestDataGenerator.writePartitionMetadata(fs, HoodieTestDataGenerator.DEFAULT_PARTITION_PATHS, tablePath);
-
-    // one commit file
-    HoodieTestUtils.createCommitFiles(tablePath, commitTime1);
-    // one .inflight commit file
-    HoodieTestUtils.createInflightCommitFiles(tablePath, commitTime2);
-
-    // generate commit files for commit 100
-    for (String commitTime : Arrays.asList(commitTime1)) {
-      HoodieTestUtils.createDataFile(tablePath, HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH, commitTime, "file-1");
-      HoodieTestUtils.createDataFile(tablePath, HoodieTestDataGenerator.DEFAULT_SECOND_PARTITION_PATH, commitTime, "file-2");
-      HoodieTestUtils.createDataFile(tablePath, HoodieTestDataGenerator.DEFAULT_THIRD_PARTITION_PATH, commitTime, "file-3");
-    }
-
-    // generate commit and marker files for inflight commit 101
-    for (String commitTime : Arrays.asList(commitTime2)) {
-      HoodieTestUtils.createDataFile(tablePath, HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH, commitTime, "file-1");
-      FileCreateUtils.createMarkerFile(tablePath, HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH, commitTime, "file-1", IOType.MERGE);
-      HoodieTestUtils.createDataFile(tablePath, HoodieTestDataGenerator.DEFAULT_SECOND_PARTITION_PATH, commitTime, "file-2");
-      FileCreateUtils.createMarkerFile(tablePath, HoodieTestDataGenerator.DEFAULT_SECOND_PARTITION_PATH, commitTime, "file-2", IOType.MERGE);
-      HoodieTestUtils.createDataFile(tablePath, HoodieTestDataGenerator.DEFAULT_THIRD_PARTITION_PATH, commitTime, "file-3");
-      FileCreateUtils.createMarkerFile(tablePath, HoodieTestDataGenerator.DEFAULT_THIRD_PARTITION_PATH, commitTime, "file-3", IOType.MERGE);
-    }
+    HoodieTestTable.of(metaClient)
+        .withPartitionMetaFiles(DEFAULT_PARTITION_PATHS)
+        .addCommit("100")
+        .withBaseFilesInPartition(DEFAULT_FIRST_PARTITION_PATH, "file-1")
+        .withBaseFilesInPartition(DEFAULT_SECOND_PARTITION_PATH, "file-2")
+        .withBaseFilesInPartition(DEFAULT_THIRD_PARTITION_PATH, "file-3")
+        .addInflightCommit("101")
+        .withBaseFilesInPartition(DEFAULT_FIRST_PARTITION_PATH, "file-1")
+        .withBaseFilesInPartition(DEFAULT_SECOND_PARTITION_PATH, "file-2")
+        .withBaseFilesInPartition(DEFAULT_THIRD_PARTITION_PATH, "file-3")
+        .withMarkerFile(DEFAULT_FIRST_PARTITION_PATH, "file-1", IOType.MERGE)
+        .withMarkerFile(DEFAULT_SECOND_PARTITION_PATH, "file-2", IOType.MERGE)
+        .withMarkerFile(DEFAULT_THIRD_PARTITION_PATH, "file-3", IOType.MERGE);
  }

  @Test
  public void testDowngradeCommand() throws Exception {
-    metaClient = HoodieTableMetaClient.reload(HoodieCLI.getTableMetaClient());
-
    // update hoodie.table.version to 1
    metaClient.getTableConfig().setTableVersion(HoodieTableVersion.ONE);
    try (FSDataOutputStream os = metaClient.getFs().create(new Path(metaClient.getMetaPath() + "/" + HoodieTableConfig.HOODIE_PROPERTIES_FILE), true)) {
@@ -98,7 +86,7 @@ public class TestUpgradeDowngradeCommand extends AbstractShellIntegrationTest {
    metaClient = HoodieTableMetaClient.reload(HoodieCLI.getTableMetaClient());

    // verify marker files for inflight commit exists
-    for (String partitionPath : HoodieTestDataGenerator.DEFAULT_PARTITION_PATHS) {
+    for (String partitionPath : DEFAULT_PARTITION_PATHS) {
      assertEquals(1, FileCreateUtils.getTotalMarkerFileCount(tablePath, partitionPath, "101", IOType.MERGE));
    }

@@ -112,7 +100,7 @@ public class TestUpgradeDowngradeCommand extends AbstractShellIntegrationTest {
    assertTableVersionFromPropertyFile();

    // verify marker files are non existant
-    for (String partitionPath : HoodieTestDataGenerator.DEFAULT_PARTITION_PATHS) {
+    for (String partitionPath : DEFAULT_PARTITION_PATHS) {
      assertEquals(0, FileCreateUtils.getTotalMarkerFileCount(tablePath, partitionPath, "101", IOType.MERGE));
    }
  }
--- a/hudi-cli/src/test/java/org/apache/hudi/cli/integ/ITTestCommitsCommand.java
+++ b/hudi-cli/src/test/java/org/apache/hudi/cli/integ/ITTestCommitsCommand.java
@@ -26,17 +26,21 @@ import org.apache.hudi.common.model.HoodieTableType;
 import org.apache.hudi.common.table.HoodieTableMetaClient;
 import org.apache.hudi.common.table.timeline.HoodieActiveTimeline;
 import org.apache.hudi.common.table.timeline.versioning.TimelineLayoutVersion;
-import org.apache.hudi.common.testutils.HoodieTestDataGenerator;
-import org.apache.hudi.common.testutils.HoodieTestUtils;
+import org.apache.hudi.common.testutils.HoodieTestTable;

 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 import org.springframework.shell.core.CommandResult;

-import java.io.File;
 import java.io.IOException;
-import java.util.Arrays;
+import java.nio.file.Paths;
+import java.util.HashMap;
+import java.util.Map;

+import static org.apache.hudi.common.testutils.HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH;
+import static org.apache.hudi.common.testutils.HoodieTestDataGenerator.DEFAULT_PARTITION_PATHS;
+import static org.apache.hudi.common.testutils.HoodieTestDataGenerator.DEFAULT_SECOND_PARTITION_PATH;
+import static org.apache.hudi.common.testutils.HoodieTestDataGenerator.DEFAULT_THIRD_PARTITION_PATH;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertTrue;

@@ -48,43 +52,45 @@ import static org.junit.jupiter.api.Assertions.assertTrue;
 */
 public class ITTestCommitsCommand extends AbstractShellIntegrationTest {

-  private String tablePath;
-
  @BeforeEach
  public void init() throws IOException {
-    String tableName = "test_table";
-    tablePath = basePath + File.separator + tableName;
+    String tableName = "test_table_" + ITTestCommitsCommand.class.getName();
+    String tablePath = Paths.get(basePath, tableName).toString();

    HoodieCLI.conf = jsc.hadoopConfiguration();
    // Create table and connect
    new TableCommand().createTable(
        tablePath, tableName, HoodieTableType.COPY_ON_WRITE.name(),
        "", TimelineLayoutVersion.VERSION_1, "org.apache.hudi.common.model.HoodieAvroPayload");
+    metaClient.setBasePath(tablePath);
+    metaClient = HoodieTableMetaClient.reload(metaClient);
  }

  /**
   * Test case of 'commit rollback' command.
   */
  @Test
-  public void testRollbackCommit() throws IOException {
+  public void testRollbackCommit() throws Exception {
    //Create some commits files and parquet files
-    String commitTime1 = "100";
-    String commitTime2 = "101";
-    String commitTime3 = "102";
-    HoodieTestDataGenerator.writePartitionMetadata(fs, HoodieTestDataGenerator.DEFAULT_PARTITION_PATHS, tablePath);
-
-    // three commit files
-    HoodieTestUtils.createCommitFiles(tablePath, commitTime1, commitTime2, commitTime3);
-
-    // generate commit files for commits
-    for (String commitTime : Arrays.asList(commitTime1, commitTime2, commitTime3)) {
-      HoodieTestUtils.createDataFile(tablePath, HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH, commitTime, "file-1");
-      HoodieTestUtils.createDataFile(tablePath, HoodieTestDataGenerator.DEFAULT_SECOND_PARTITION_PATH, commitTime, "file-2");
-      HoodieTestUtils.createDataFile(tablePath, HoodieTestDataGenerator.DEFAULT_THIRD_PARTITION_PATH, commitTime, "file-3");
-    }
+    Map<String, String> partitionAndFileId = new HashMap<String, String>() {
+      {
+        put(DEFAULT_FIRST_PARTITION_PATH, "file-1");
+        put(DEFAULT_SECOND_PARTITION_PATH, "file-2");
+        put(DEFAULT_THIRD_PARTITION_PATH, "file-3");
+      }
+    };
+    final String rollbackCommit = "102";
+    HoodieTestTable.of(metaClient)
+        .withPartitionMetaFiles(DEFAULT_PARTITION_PATHS)
+        .addCommit("100")
+        .withBaseFilesInPartitions(partitionAndFileId)
+        .addCommit("101")
+        .withBaseFilesInPartitions(partitionAndFileId)
+        .addCommit(rollbackCommit)
+        .withBaseFilesInPartitions(partitionAndFileId);

    CommandResult cr = getShell().executeCommand(String.format("commit rollback --commit %s --sparkMaster %s --sparkMemory %s",
-        commitTime3, "local", "4G"));
+        rollbackCommit, "local", "4G"));
    assertTrue(cr.isSuccess());

    metaClient = HoodieTableMetaClient.reload(HoodieCLI.getTableMetaClient());
--- a/hudi-cli/src/test/java/org/apache/hudi/cli/testutils/HoodieTestCommitMetadataGenerator.java
+++ b/hudi-cli/src/test/java/org/apache/hudi/cli/testutils/HoodieTestCommitMetadataGenerator.java
@@ -23,18 +23,14 @@ import org.apache.hudi.common.model.HoodieCommitMetadata;
 import org.apache.hudi.common.model.HoodieWriteStat;
 import org.apache.hudi.common.table.HoodieTableMetaClient;
 import org.apache.hudi.common.table.timeline.HoodieTimeline;
+import org.apache.hudi.common.testutils.FileCreateUtils;
 import org.apache.hudi.common.testutils.HoodieTestDataGenerator;
-import org.apache.hudi.common.testutils.HoodieTestUtils;
-import org.apache.hudi.common.util.CollectionUtils;
 import org.apache.hudi.common.util.Option;
-import org.apache.hudi.exception.HoodieIOException;

 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;

-import java.io.IOException;
 import java.nio.charset.StandardCharsets;
 import java.util.Arrays;
 import java.util.HashMap;
@@ -42,6 +38,9 @@ import java.util.List;
 import java.util.Map;
 import java.util.UUID;

+import static org.apache.hudi.common.testutils.FileCreateUtils.baseFileName;
+import static org.apache.hudi.common.util.CollectionUtils.createImmutableList;
+
 /**
 * Class to be used in tests to keep generating test inserts and updates against a corpus.
 */
@@ -62,67 +61,53 @@ public class HoodieTestCommitMetadataGenerator extends HoodieTestDataGenerator {
  /**
   * Create a commit file with default CommitMetadata.
   */
-  public static void createCommitFileWithMetadata(String basePath, String commitTime, Configuration configuration) {
+  public static void createCommitFileWithMetadata(String basePath, String commitTime, Configuration configuration) throws Exception {
    createCommitFileWithMetadata(basePath, commitTime, configuration, Option.empty(), Option.empty());
  }

  public static void createCommitFileWithMetadata(String basePath, String commitTime, Configuration configuration,
-      Option<Integer> writes, Option<Integer> updates) {
+      Option<Integer> writes, Option<Integer> updates) throws Exception {
    createCommitFileWithMetadata(basePath, commitTime, configuration, UUID.randomUUID().toString(),
        UUID.randomUUID().toString(), writes, updates);
  }

  public static void createCommitFileWithMetadata(String basePath, String commitTime, Configuration configuration,
-      String fileId1, String fileId2, Option<Integer> writes, Option<Integer> updates) {
-    Arrays.asList(HoodieTimeline.makeCommitFileName(commitTime), HoodieTimeline.makeInflightCommitFileName(commitTime),
-        HoodieTimeline.makeRequestedCommitFileName(commitTime))
-        .forEach(f -> {
-          Path commitFile = new Path(
-              basePath + "/" + HoodieTableMetaClient.METAFOLDER_NAME + "/" + f);
-          FSDataOutputStream os = null;
-          try {
-            FileSystem fs = FSUtils.getFs(basePath, configuration);
-            os = fs.create(commitFile, true);
-            // Generate commitMetadata
-            HoodieCommitMetadata commitMetadata =
-                generateCommitMetadata(basePath, commitTime, fileId1, fileId2, writes, updates);
-            // Write empty commit metadata
-            os.writeBytes(new String(commitMetadata.toJsonString().getBytes(StandardCharsets.UTF_8)));
-          } catch (IOException ioe) {
-            throw new HoodieIOException(ioe.getMessage(), ioe);
-          } finally {
-            if (null != os) {
-              try {
-                os.close();
-              } catch (IOException e) {
-                throw new HoodieIOException(e.getMessage(), e);
-              }
-            }
-          }
-        });
+      String fileId1, String fileId2, Option<Integer> writes, Option<Integer> updates) throws Exception {
+    List<String> commitFileNames = Arrays.asList(HoodieTimeline.makeCommitFileName(commitTime), HoodieTimeline.makeInflightCommitFileName(commitTime),
+        HoodieTimeline.makeRequestedCommitFileName(commitTime));
+    for (String name : commitFileNames) {
+      Path commitFilePath = new Path(basePath + "/" + HoodieTableMetaClient.METAFOLDER_NAME + "/" + name);
+      try (FSDataOutputStream os = FSUtils.getFs(basePath, configuration).create(commitFilePath, true)) {
+        // Generate commitMetadata
+        HoodieCommitMetadata commitMetadata =
+            generateCommitMetadata(basePath, commitTime, fileId1, fileId2, writes, updates);
+        // Write empty commit metadata
+        os.writeBytes(new String(commitMetadata.toJsonString().getBytes(StandardCharsets.UTF_8)));
+      }
+    }
  }

  /**
   * Generate commitMetadata in path.
   */
-  public static HoodieCommitMetadata generateCommitMetadata(String basePath, String commitTime) throws IOException {
+  public static HoodieCommitMetadata generateCommitMetadata(String basePath, String commitTime) throws Exception {
    return generateCommitMetadata(basePath, commitTime, Option.empty(), Option.empty());
  }

  public static HoodieCommitMetadata generateCommitMetadata(String basePath, String commitTime,
-      Option<Integer> writes, Option<Integer> updates) throws IOException {
+      Option<Integer> writes, Option<Integer> updates) throws Exception {
    return generateCommitMetadata(basePath, commitTime, UUID.randomUUID().toString(), UUID.randomUUID().toString(),
        writes, updates);
  }

  public static HoodieCommitMetadata generateCommitMetadata(String basePath, String commitTime, String fileId1,
-      String fileId2, Option<Integer> writes, Option<Integer> updates) throws IOException {
-    String file1P0C0 = HoodieTestUtils.createDataFile(basePath, DEFAULT_FIRST_PARTITION_PATH, commitTime, fileId1);
-    String file1P1C0 = HoodieTestUtils.createDataFile(basePath, DEFAULT_SECOND_PARTITION_PATH, commitTime, fileId2);
+      String fileId2, Option<Integer> writes, Option<Integer> updates) throws Exception {
+    FileCreateUtils.createBaseFile(basePath, DEFAULT_FIRST_PARTITION_PATH, commitTime, fileId1);
+    FileCreateUtils.createBaseFile(basePath, DEFAULT_SECOND_PARTITION_PATH, commitTime, fileId2);
    return generateCommitMetadata(new HashMap<String, List<String>>() {
      {
-        put(DEFAULT_FIRST_PARTITION_PATH, CollectionUtils.createImmutableList(file1P0C0));
-        put(DEFAULT_SECOND_PARTITION_PATH, CollectionUtils.createImmutableList(file1P1C0));
+        put(DEFAULT_FIRST_PARTITION_PATH, createImmutableList(baseFileName(DEFAULT_FIRST_PARTITION_PATH, fileId1)));
+        put(DEFAULT_SECOND_PARTITION_PATH, createImmutableList(baseFileName(DEFAULT_SECOND_PARTITION_PATH, fileId2)));
      }
    }, writes, updates);
  }