From 30ebdc708bcac5e08c7b1352d726d1c2909f548f Mon Sep 17 00:00:00 2001 From: xi chaomin <36392121+xicm@users.noreply.github.com> Date: Fri, 24 Jun 2022 14:47:36 +0800 Subject: [PATCH] [HUDI-3735] TestHoodieSparkMergeOnReadTableRollback is flaky (#5874) --- .../TestHoodieSparkMergeOnReadTableRollback.java | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/functional/TestHoodieSparkMergeOnReadTableRollback.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/functional/TestHoodieSparkMergeOnReadTableRollback.java index 043697f66..c33f0bcc2 100644 --- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/functional/TestHoodieSparkMergeOnReadTableRollback.java +++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/functional/TestHoodieSparkMergeOnReadTableRollback.java @@ -227,8 +227,8 @@ public class TestHoodieSparkMergeOnReadTableRollback extends SparkClientFunction allFiles = listAllBaseFilesInPath(hoodieTable); // After rollback, there should be no base file with the failed commit time List remainingFiles = Arrays.stream(allFiles).filter(file -> file.getPath().getName() - .contains(commitTime1)).map(fileStatus -> fileStatus.getPath().toString()).collect(Collectors.toList()); - assertEquals(0, remainingFiles.size(), "There files should have been rolled-back " + .contains("_" + commitTime1)).map(fileStatus -> fileStatus.getPath().toString()).collect(Collectors.toList()); + assertEquals(0, remainingFiles.size(), "These files should have been rolled-back " + "when rolling back commit " + commitTime1 + " but are still remaining. Files: " + remainingFiles); inputPaths = tableView.getLatestBaseFiles() .map(baseFile -> new Path(baseFile.getPath()).getParent().toString()) @@ -240,7 +240,7 @@ public class TestHoodieSparkMergeOnReadTableRollback extends SparkClientFunction /* * Write 3 (inserts + updates - testing successful delta commit) */ - final String commitTime2 = "003"; + final String commitTime2 = "000000003"; try (SparkRDDWriteClient thirdClient = getHoodieWriteClient(getHoodieWriteConfigWithSmallFileHandlingOff(true));) { thirdClient.startCommitWithTime(commitTime2); @@ -265,8 +265,10 @@ public class TestHoodieSparkMergeOnReadTableRollback extends SparkClientFunction thirdClient.rollback(commitTime2); allFiles = listAllBaseFilesInPath(hoodieTable); // After rollback, there should be no base file with the failed commit time - assertEquals(0, Arrays.stream(allFiles) - .filter(file -> file.getPath().getName().contains(commitTime2)).count()); + List remainingFiles = Arrays.stream(allFiles).filter(file -> file.getPath().getName() + .contains("_" + commitTime2)).map(fileStatus -> fileStatus.getPath().toString()).collect(Collectors.toList()); + assertEquals(0, remainingFiles.size(), "These files should have been rolled-back " + + "when rolling back commit " + commitTime2 + " but are still remaining. Files: " + remainingFiles); metaClient = HoodieTableMetaClient.reload(metaClient); hoodieTable = HoodieSparkTable.create(cfg, context(), metaClient); @@ -282,7 +284,7 @@ public class TestHoodieSparkMergeOnReadTableRollback extends SparkClientFunction /* * Write 4 (updates) */ - newCommitTime = "004"; + newCommitTime = "000000004"; thirdClient.startCommitWithTime(newCommitTime); writeStatusJavaRDD = thirdClient.upsert(writeRecords, newCommitTime);