lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

Ensure Hoodie metadata folder and files are filtered out when constructing Parquet Data Source

Browse Source
This commit is contained in:
Balaji Varadarajan
2018-09-28 21:41:28 -07:00
committed by vinoth chandar
parent 06bdba3cef
commit 9710b5a3a6
3 changed files with 35 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
hoodie-hadoop-mr/src/test/java/com/uber/hoodie/hadoop/TestHoodieROTablePathFilter.java
View File

@@ -23,6 +23,7 @@ import com.uber.hoodie.common.model.HoodieTestUtils;
import com.uber.hoodie.common.table.HoodieTableMetaClient;
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import org.apache.hadoop.fs.Path;
import org.junit.Before;
import org.junit.Rule;
@@ -51,6 +52,7 @@ public class TestHoodieROTablePathFilter {
HoodieTestUtils.createCommitFiles(basePath, "001", "002");
HoodieTestUtils.createInflightCommitFiles(basePath, "003");
HoodieTestUtils.createCompactionRequest(metaClient, "004", new ArrayList<>());
HoodieTestUtils.createDataFile(basePath, "2017/01/01", "001", "f1");
HoodieTestUtils.createDataFile(basePath, "2017/01/01", "001", "f2");
@@ -72,6 +74,19 @@ public class TestHoodieROTablePathFilter {
"file:///" + HoodieTestUtils.getDataFilePath(basePath, "2017/01/01", "002", "f2"))));
assertFalse(pathFilter.accept(new Path(
"file:///" + HoodieTestUtils.getDataFilePath(basePath, "2017/01/01", "003", "f3"))));
assertFalse(pathFilter.accept(new Path("file:///" + HoodieTestUtils.getCommitFilePath(basePath, "001"))));
assertFalse(pathFilter.accept(new Path("file:///" + HoodieTestUtils.getCommitFilePath(basePath, "002"))));
assertFalse(pathFilter.accept(new Path("file:///"
+ HoodieTestUtils.getInflightCommitFilePath(basePath, "003"))));
assertFalse(pathFilter.accept(new Path("file:///"
+ HoodieTestUtils.getRequestedCompactionFilePath(basePath, "004"))));
assertFalse(pathFilter.accept(new Path("file:///" + basePath + "/"
+ HoodieTableMetaClient.METAFOLDER_NAME + "/")));
assertFalse(pathFilter.accept(new Path("file:///" + basePath + "/" + HoodieTableMetaClient.METAFOLDER_NAME)));
assertFalse(pathFilter.accept(new Path(
"file:///" + HoodieTestUtils.getDataFilePath(basePath, "2017/01/01", "003", "f3"))));
}
@Test