lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-2952] Fixing metadata table for non-partitioned dataset (#4243)

Browse Source
This commit is contained in:
Sivabalan Narayanan
2021-12-10 11:11:42 -05:00
committed by GitHub
parent f194566ed4
commit be368264f4
9 changed files with 104 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
hudi-common/src/test/java/org/apache/hudi/common/testutils/FileCreateUtils.java
View File

@@ -367,7 +367,9 @@ public class FileCreateUtils {
if (Files.notExists(basePath)) {
return Collections.emptyList();
}
return Files.list(basePath).filter(entry -> !entry.getFileName().toString().equals(HoodieTableMetaClient.METAFOLDER_NAME)).collect(Collectors.toList());
return Files.list(basePath).filter(entry -> (!entry.getFileName().toString().equals(HoodieTableMetaClient.METAFOLDER_NAME)
&& !entry.getFileName().toString().contains("parquet") && !entry.getFileName().toString().contains("log"))
&& !entry.getFileName().toString().endsWith(HoodiePartitionMetadata.HOODIE_PARTITION_METAFILE)).collect(Collectors.toList());
}
/**

6
hudi-common/src/test/java/org/apache/hudi/common/testutils/HoodieTestTable.java
View File

@@ -602,7 +602,7 @@ public class HoodieTestTable {
}
public List<java.nio.file.Path> getAllPartitionPaths() throws IOException {
java.nio.file.Path basePathPath = Paths.get(basePath, HoodieTableMetaClient.TEMPFOLDER_NAME).getParent().getParent();
java.nio.file.Path basePathPath = Paths.get(basePath);
return FileCreateUtils.getPartitionPaths(basePathPath);
}
@@ -660,8 +660,10 @@ public class HoodieTestTable {
return FileSystemTestUtils.listRecursive(fs, new Path(Paths.get(basePath, partitionPath).toString())).stream()
.filter(entry -> {
boolean toReturn = true;
String filePath = entry.getPath().toString();
String fileName = entry.getPath().getName();
if (fileName.equals(HoodiePartitionMetadata.HOODIE_PARTITION_METAFILE)) {
if (fileName.equals(HoodiePartitionMetadata.HOODIE_PARTITION_METAFILE) || (!fileName.contains("log") && !fileName.contains("parquet"))
|| filePath.contains("metadata")) {
toReturn = false;
} else {
for (String inflight : inflightCommits) {