lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-988] Fix Unit Test Flakiness : Ensure all instantiations of HoodieWriteClient is closed properly. Fix bug in TestRollbacks. Make CLI unit tests for Hudi CLI check skip redering strings

Browse Source
This commit is contained in:
Balaji Varadarajan
2020-06-02 01:49:37 -07:00
committed by Balaji Varadarajan
parent a9a97d6af4
commit a68180b179
26 changed files with 160 additions and 121 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestArchivedCommitsCommand.java
View File

@@ -146,7 +146,9 @@ public class TestArchivedCommitsCommand extends AbstractShellIntegrationTest {
String expectedResult = HoodiePrintHelper.print(
header, new HashMap<>(), "", false, -1, false, rows);
assertEquals(expectedResult, cr.getResult().toString());
expectedResult = removeNonWordAndStripSpace(expectedResult);
String got = removeNonWordAndStripSpace(cr.getResult().toString());
assertEquals(expectedResult, got);
}
/**
@@ -169,7 +171,9 @@ public class TestArchivedCommitsCommand extends AbstractShellIntegrationTest {
}
rows.add(new Comparable[] {"103", "commit"});
String expected = HoodiePrintHelper.print(header, new HashMap<>(), "", false, 10, false, rows);
assertEquals(expected, cr.getResult().toString());
expected = removeNonWordAndStripSpace(expected);
String got = removeNonWordAndStripSpace(cr.getResult().toString());
assertEquals(expected, got);
// Test with Metadata and no limit
cr = getShell().executeCommand("show archived commits --skipMetadata false --limit -1");
@@ -189,6 +193,8 @@ public class TestArchivedCommitsCommand extends AbstractShellIntegrationTest {
}
header = header.addTableHeaderField("CommitDetails");
expected = HoodiePrintHelper.print(header, new HashMap<>(), "", false, -1, false, rows);
assertEquals(expected, cr.getResult().toString());
expected = removeNonWordAndStripSpace(expected);
got = removeNonWordAndStripSpace(cr.getResult().toString());
assertEquals(expected, got);
}
}

8
hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestCleansCommand.java
View File

@@ -128,7 +128,9 @@ public class TestCleansCommand extends AbstractShellIntegrationTest {
rows.add(new Comparable[] {clean.getTimestamp(), "102", "0", getLatestCleanTimeTakenInMillis().toString()});
String expected = HoodiePrintHelper.print(header, new HashMap<>(), "", false, -1, false, rows);
assertEquals(expected, cr.getResult().toString());
expected = removeNonWordAndStripSpace(expected);
String got = removeNonWordAndStripSpace(cr.getResult().toString());
assertEquals(expected, got);
}
/**
@@ -168,7 +170,9 @@ public class TestCleansCommand extends AbstractShellIntegrationTest {
HoodieCleaningPolicy.KEEP_LATEST_COMMITS, "0", "0"});
String expected = HoodiePrintHelper.print(header, new HashMap<>(), "", false, -1, false, rows);
assertEquals(expected, cr.getResult().toString());
expected = removeNonWordAndStripSpace(expected);
String got = removeNonWordAndStripSpace(cr.getResult().toString());
assertEquals(expected, got);
}
/**

12
hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestFileSystemViewCommand.java
View File

@@ -147,7 +147,9 @@ public class TestFileSystemViewCommand extends AbstractShellIntegrationTest {
.addTableHeaderField(HoodieTableHeaderFields.HEADER_TOTAL_DELTA_FILE_SIZE)
.addTableHeaderField(HoodieTableHeaderFields.HEADER_DELTA_FILES);
String expected = HoodiePrintHelper.print(header, fieldNameToConverterMap, "", false, -1, false, rows);
assertEquals(expected, cr.getResult().toString());
expected = removeNonWordAndStripSpace(expected);
String got = removeNonWordAndStripSpace(cr.getResult().toString());
assertEquals(expected, got);
}
/**
@@ -188,7 +190,9 @@ public class TestFileSystemViewCommand extends AbstractShellIntegrationTest {
.addTableHeaderField(HoodieTableHeaderFields.HEADER_DATA_FILE_SIZE);
String expected = HoodiePrintHelper.print(header, fieldNameToConverterMap, "", false, -1, false, rows);
assertEquals(expected, cr.getResult().toString());
expected = removeNonWordAndStripSpace(expected);
String got = removeNonWordAndStripSpace(cr.getResult().toString());
assertEquals(expected, got);
}
/**
@@ -262,6 +266,8 @@ public class TestFileSystemViewCommand extends AbstractShellIntegrationTest {
.addTableHeaderField(HoodieTableHeaderFields.HEADER_DELTA_FILES_SCHEDULED)
.addTableHeaderField(HoodieTableHeaderFields.HEADER_DELTA_FILES_UNSCHEDULED);
String expected = HoodiePrintHelper.print(header, fieldNameToConverterMap, "", false, -1, false, rows);
assertEquals(expected, cr.getResult().toString());
expected = removeNonWordAndStripSpace(expected);
String got = removeNonWordAndStripSpace(cr.getResult().toString());
assertEquals(expected, got);
}
}

15
hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestHoodieLogFileCommand.java
View File

@@ -135,8 +135,9 @@ public class TestHoodieLogFileCommand extends AbstractShellIntegrationTest {
rows.add(output);
String expected = HoodiePrintHelper.print(header, new HashMap<>(), "", false, -1, false, rows);
assertEquals(expected, cr.getResult().toString());
expected = removeNonWordAndStripSpace(expected);
String got = removeNonWordAndStripSpace(cr.getResult().toString());
assertEquals(expected, got);
}
/**
@@ -151,8 +152,9 @@ public class TestHoodieLogFileCommand extends AbstractShellIntegrationTest {
List<IndexedRecord> records = SchemaTestUtil.generateTestRecords(0, 10);
String[][] rows = records.stream().map(r -> new String[]{r.toString()}).toArray(String[][]::new);
String expected = HoodiePrintHelper.print(new String[] {HoodieTableHeaderFields.HEADER_RECORDS}, rows);
assertEquals(expected, cr.getResult().toString());
expected = removeNonWordAndStripSpace(expected);
String got = removeNonWordAndStripSpace(cr.getResult().toString());
assertEquals(expected, got);
}
/**
@@ -216,7 +218,8 @@ public class TestHoodieLogFileCommand extends AbstractShellIntegrationTest {
assertNotNull(rows);
String expected = HoodiePrintHelper.print(new String[] {HoodieTableHeaderFields.HEADER_RECORDS}, rows);
assertEquals(expected, cr.getResult().toString());
expected = removeNonWordAndStripSpace(expected);
String got = removeNonWordAndStripSpace(cr.getResult().toString());
assertEquals(expected, got);
}
}

19
hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestRepairsCommand.java
View File

@@ -96,8 +96,9 @@ public class TestRepairsCommand extends AbstractShellIntegrationTest {
.toArray(String[][]::new);
String expected = HoodiePrintHelper.print(new String[] {HoodieTableHeaderFields.HEADER_PARTITION_PATH,
HoodieTableHeaderFields.HEADER_METADATA_PRESENT, HoodieTableHeaderFields.HEADER_REPAIR_ACTION}, rows);
assertEquals(expected, cr.getResult().toString());
expected = removeNonWordAndStripSpace(expected);
String got = removeNonWordAndStripSpace(cr.getResult().toString());
assertEquals(expected, got);
}
/**
@@ -126,8 +127,9 @@ public class TestRepairsCommand extends AbstractShellIntegrationTest {
.toArray(String[][]::new);
String expected = HoodiePrintHelper.print(new String[] {HoodieTableHeaderFields.HEADER_PARTITION_PATH,
HoodieTableHeaderFields.HEADER_METADATA_PRESENT, HoodieTableHeaderFields.HEADER_REPAIR_ACTION}, rows);
assertEquals(expected, cr.getResult().toString());
expected = removeNonWordAndStripSpace(expected);
String got = removeNonWordAndStripSpace(cr.getResult().toString());
assertEquals(expected, got);
cr = getShell().executeCommand("repair addpartitionmeta");
@@ -137,7 +139,9 @@ public class TestRepairsCommand extends AbstractShellIntegrationTest {
.toArray(String[][]::new);
expected = HoodiePrintHelper.print(new String[] {HoodieTableHeaderFields.HEADER_PARTITION_PATH,
HoodieTableHeaderFields.HEADER_METADATA_PRESENT, HoodieTableHeaderFields.HEADER_REPAIR_ACTION}, rows);
assertEquals(expected, cr.getResult().toString());
expected = removeNonWordAndStripSpace(expected);
got = removeNonWordAndStripSpace(cr.getResult().toString());
assertEquals(expected, got);
}
/**
@@ -170,8 +174,9 @@ public class TestRepairsCommand extends AbstractShellIntegrationTest {
.toArray(String[][]::new);
String expect = HoodiePrintHelper.print(new String[] {HoodieTableHeaderFields.HEADER_HOODIE_PROPERTY,
HoodieTableHeaderFields.HEADER_OLD_VALUE, HoodieTableHeaderFields.HEADER_NEW_VALUE}, rows);
assertEquals(expect, cr.getResult().toString());
expect = removeNonWordAndStripSpace(expect);
String got = removeNonWordAndStripSpace(cr.getResult().toString());
assertEquals(expect, got);
}
/**

14
hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestRollbacksCommand.java
View File

@@ -88,7 +88,7 @@ public class TestRollbacksCommand extends AbstractShellIntegrationTest {
HoodieWriteConfig config = HoodieWriteConfig.newBuilder().withPath(tablePath)
.withIndexConfig(HoodieIndexConfig.newBuilder().withIndexType(HoodieIndex.IndexType.INMEMORY).build()).build();
try (HoodieWriteClient client = new HoodieWriteClient(jsc, config, false)) {
try (HoodieWriteClient client = getHoodieWriteClient(config)) {
// Rollback inflight commit3 and commit2
client.rollback(commitTime3);
client.rollback(commitTime2);
@@ -119,7 +119,7 @@ public class TestRollbacksCommand extends AbstractShellIntegrationTest {
row[1] = c;
// expect data
row[2] = 3;
row[3] = 0;
row[3] = metadata.getTimeTakenInMillis();
row[4] = 3;
rows.add(row);
});
@@ -134,8 +134,9 @@ public class TestRollbacksCommand extends AbstractShellIntegrationTest {
.addTableHeaderField(HoodieTableHeaderFields.HEADER_TIME_TOKEN_MILLIS)
.addTableHeaderField(HoodieTableHeaderFields.HEADER_TOTAL_PARTITIONS);
String expected = HoodiePrintHelper.print(header, new HashMap<>(), "", false, -1, false, rows);
assertEquals(expected, cr.getResult().toString());
expected = removeNonWordAndStripSpace(expected);
String got = removeNonWordAndStripSpace(cr.getResult().toString());
assertEquals(expected, got);
}
/**
@@ -176,7 +177,8 @@ public class TestRollbacksCommand extends AbstractShellIntegrationTest {
.addTableHeaderField(HoodieTableHeaderFields.HEADER_DELETED_FILE)
.addTableHeaderField(HoodieTableHeaderFields.HEADER_SUCCEEDED);
String expected = HoodiePrintHelper.print(header, new HashMap<>(), "", false, -1, false, rows);
assertEquals(expected, cr.getResult().toString());
expected = removeNonWordAndStripSpace(expected);
String got = removeNonWordAndStripSpace(cr.getResult().toString());
assertEquals(expected, got);
}
}

5
hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestSavepointsCommand.java
View File

@@ -75,8 +75,9 @@ public class TestSavepointsCommand extends AbstractShellIntegrationTest {
String[][] rows = Arrays.asList("100", "101", "102", "103").stream().sorted(Comparator.reverseOrder())
.map(instant -> new String[]{instant}).toArray(String[][]::new);
String expected = HoodiePrintHelper.print(new String[] {HoodieTableHeaderFields.HEADER_SAVEPOINT_TIME}, rows);
assertEquals(expected, cr.getResult().toString());
expected = removeNonWordAndStripSpace(expected);
String got = removeNonWordAndStripSpace(cr.getResult().toString());
assertEquals(expected, got);
}
/**

8
hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestSparkEnvCommand.java
View File

@@ -40,7 +40,9 @@ public class TestSparkEnvCommand extends AbstractShellIntegrationTest {
// First, be empty
CommandResult cr = getShell().executeCommand("show envs all");
String nullResult = HoodiePrintHelper.print(new String[] {"key", "value"}, new String[0][2]);
assertEquals(nullResult, cr.getResult().toString());
nullResult = removeNonWordAndStripSpace(nullResult);
String got = removeNonWordAndStripSpace(cr.getResult().toString());
assertEquals(nullResult, got);
// Set SPARK_HOME
cr = getShell().executeCommand("set --conf SPARK_HOME=/usr/etc/spark");
@@ -49,6 +51,8 @@ public class TestSparkEnvCommand extends AbstractShellIntegrationTest {
//Get
cr = getShell().executeCommand("show env --key SPARK_HOME");
String result = HoodiePrintHelper.print(new String[] {"key", "value"}, new String[][] {new String[] {"SPARK_HOME", "/usr/etc/spark"}});
assertEquals(result, cr.getResult().toString());
result = removeNonWordAndStripSpace(result);
got = removeNonWordAndStripSpace(cr.getResult().toString());
assertEquals(result, got);
}
}

10
hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestStatsCommand.java
View File

@@ -103,8 +103,9 @@ public class TestStatsCommand extends AbstractShellIntegrationTest {
.addTableHeaderField(HoodieTableHeaderFields.HEADER_TOTAL_WRITTEN)
.addTableHeaderField(HoodieTableHeaderFields.HEADER_WRITE_AMPLIFICATION_FACTOR);
String expected = HoodiePrintHelper.print(header, new HashMap<>(), "", false, -1, false, rows);
assertEquals(expected, cr.getResult().toString());
expected = removeNonWordAndStripSpace(expected);
String got = removeNonWordAndStripSpace(cr.getResult().toString());
assertEquals(expected, got);
}
/**
@@ -170,7 +171,8 @@ public class TestStatsCommand extends AbstractShellIntegrationTest {
.addTableHeaderField(HoodieTableHeaderFields.HEADER_HISTOGRAM_STD_DEV);
String expect = HoodiePrintHelper.print(header, new StatsCommand().getFieldNameToConverterMap(),
"", false, -1, false, rows);
assertEquals(expect, cr.getResult().toString());
expect = removeNonWordAndStripSpace(expect);
String got = removeNonWordAndStripSpace(cr.getResult().toString());
assertEquals(expect, got);
}
}

9
hudi-cli/src/test/java/org/apache/hudi/cli/testutils/AbstractShellIntegrationTest.java
View File

@@ -58,4 +58,13 @@ public abstract class AbstractShellIntegrationTest extends HoodieClientTestHarne
protected static JLineShellComponent getShell() {
return shell;
}
/**
* Helper to prepare string for matching.
* @param str Input string.
* @return pruned string with non word characters removed.
*/
protected String removeNonWordAndStripSpace(String str) {
return str.replaceAll("[\\s]+", ",").replaceAll("[\\W]+", ",");
}
}