lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[MINOR] Fixing integ test suite for hudi-aws and archival validation (#4142)

Browse Source
This commit is contained in:
Sivabalan Narayanan
2021-11-28 20:11:50 -05:00
committed by GitHub
parent eca1693288
commit 52aae36b53
2 changed files with 13 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files

28
hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/dag/nodes/ValidateAsyncOperations.java
View File

@@ -84,29 +84,25 @@ public class ValidateAsyncOperations extends DagNode<Option<String>> {
String metadataPath = executionContext.getHoodieTestSuiteWriter().getCfg().targetBasePath + "/.hoodie"; String metadataPath = executionContext.getHoodieTestSuiteWriter().getCfg().targetBasePath + "/.hoodie";
FileStatus[] metaFileStatuses = fs.listStatus(new Path(metadataPath)); FileStatus[] metaFileStatuses = fs.listStatus(new Path(metadataPath));
boolean archFound = false;
boolean cleanFound = false; boolean cleanFound = false;
for (FileStatus fileStatus : metaFileStatuses) {
Matcher cleanFileMatcher = CLEAN_FILE_PATTERN.matcher(fileStatus.getPath().getName());
if (cleanFileMatcher.matches()) {
cleanFound = true;
break;
}
}
String archivalPath = executionContext.getHoodieTestSuiteWriter().getCfg().targetBasePath + "/.hoodie/archived";
metaFileStatuses = fs.listStatus(new Path(archivalPath));
boolean archFound = false;
for (FileStatus fileStatus : metaFileStatuses) { for (FileStatus fileStatus : metaFileStatuses) {
Matcher archFileMatcher = ARCHIVE_FILE_PATTERN.matcher(fileStatus.getPath().getName()); Matcher archFileMatcher = ARCHIVE_FILE_PATTERN.matcher(fileStatus.getPath().getName());
if (archFileMatcher.matches()) { if (archFileMatcher.matches()) {
archFound = true; archFound = true;
if (config.validateArchival() && !config.validateClean()) {
break;
}
}
Matcher cleanFileMatcher = CLEAN_FILE_PATTERN.matcher(fileStatus.getPath().getName());
if (cleanFileMatcher.matches()) {
cleanFound = true;
if (!config.validateArchival() && config.validateClean()) {
break;
}
}
if (config.validateClean() && config.validateArchival()) {
if (archFound && cleanFound) {
break;
}
} }
} }
if (config.validateArchival() && !archFound) { if (config.validateArchival() && !archFound) {
throw new AssertionError("Archival NotFound in " + metadataPath); throw new AssertionError("Archival NotFound in " + metadataPath);
} }

1
packaging/hudi-integ-test-bundle/pom.xml
View File

@@ -78,6 +78,7 @@
<include>org.apache.hudi:hudi-sync-common</include> <include>org.apache.hudi:hudi-sync-common</include>
<include>org.apache.hudi:hudi-hadoop-mr</include> <include>org.apache.hudi:hudi-hadoop-mr</include>
<include>org.apache.hudi:hudi-timeline-service</include> <include>org.apache.hudi:hudi-timeline-service</include>
<include>org.apache.hudi:hudi-aws</include>
<include>org.apache.hudi:hudi-integ-test</include> <include>org.apache.hudi:hudi-integ-test</include>
<include>org.jetbrains.kotlin:kotlin-stdlib-jdk8</include> <include>org.jetbrains.kotlin:kotlin-stdlib-jdk8</include>