lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-3643] Fix hive count exception when the table is empty and the path depth is less than 3 (#5051)

Browse Source
This commit is contained in:
董可伦
2022-04-07 19:21:03 +08:00
committed by GitHub
parent 9d744bb35c
commit 6a8396420c
5 changed files with 63 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
hudi-hadoop-mr/src/test/java/org/apache/hudi/hadoop/TestHoodieHFileInputFormat.java
View File

@@ -162,6 +162,21 @@ public class TestHoodieHFileInputFormat {
assertEquals(10, inputSplits.length);
}
@Test
public void testInputFormatLoadWithEmptyTable() throws IOException {
// initial hoodie table
String bathPathStr = "/tmp/test_empty_table";
HoodieTestUtils.init(HoodieTestUtils.getDefaultHadoopConf(), bathPathStr, HoodieTableType.COPY_ON_WRITE,
baseFileFormat);
// Add the paths
FileInputFormat.setInputPaths(jobConf, bathPathStr);
FileStatus[] files = inputFormat.listStatus(jobConf);
assertEquals(0, files.length);
InputSplit[] inputSplits = inputFormat.getSplits(jobConf, 0);
assertEquals(0, inputSplits.length);
}
@Test
public void testInputFormatUpdates() throws IOException {
// initial commit

15
hudi-hadoop-mr/src/test/java/org/apache/hudi/hadoop/TestHoodieParquetInputFormat.java
View File

@@ -167,6 +167,21 @@ public class TestHoodieParquetInputFormat {
assertEquals(10, files.length);
}
@Test
public void testInputFormatLoadWithEmptyTable() throws IOException {
// initial hoodie table
String bathPathStr = "/tmp/test_empty_table";
HoodieTestUtils.init(HoodieTestUtils.getDefaultHadoopConf(), bathPathStr, HoodieTableType.COPY_ON_WRITE,
baseFileFormat);
// Add the paths
FileInputFormat.setInputPaths(jobConf, bathPathStr);
FileStatus[] files = inputFormat.listStatus(jobConf);
assertEquals(0, files.length);
InputSplit[] inputSplits = inputFormat.getSplits(jobConf, 0);
assertEquals(0, inputSplits.length);
}
@Test
public void testInputFormatUpdates() throws IOException {
// initial commit

18
hudi-hadoop-mr/src/test/java/org/apache/hudi/hadoop/TestInputPathHandler.java
View File

@@ -56,6 +56,12 @@ public class TestInputPathHandler {
// non Hoodie table
public static final String TRIPS_STATS_TEST_NAME = "trips_stats";
// empty snapshot table
public static final String EMPTY_SNAPSHOT_TEST_NAME = "empty_snapshot";
// empty incremental table
public static final String EMPTY_INCREMENTAL_TEST_NAME = "empty_incremental";
@TempDir
static java.nio.file.Path parentPath;
@@ -67,6 +73,8 @@ public class TestInputPathHandler {
private static String basePathTable2 = null;
private static String basePathTable3 = null;
private static String basePathTable4 = null; // non hoodie Path
private static String basePathTable5 = null;
private static String basePathTable6 = null;
private static List<String> incrementalTables;
private static List<Path> incrementalPaths;
private static List<Path> snapshotPaths;
@@ -110,6 +118,9 @@ public class TestInputPathHandler {
basePathTable2 = parentPath.resolve(MODEL_TRIPS_TEST_NAME).toAbsolutePath().toString();
basePathTable3 = parentPath.resolve(ETL_TRIPS_TEST_NAME).toAbsolutePath().toString();
basePathTable4 = parentPath.resolve(TRIPS_STATS_TEST_NAME).toAbsolutePath().toString();
String tempPath = "/tmp/";
basePathTable5 = tempPath + EMPTY_SNAPSHOT_TEST_NAME;
basePathTable6 = tempPath + EMPTY_INCREMENTAL_TEST_NAME;
dfs.mkdirs(new Path(basePathTable1));
initTableType(dfs.getConf(), basePathTable1, RAW_TRIPS_TEST_NAME, HoodieTableType.MERGE_ON_READ);
@@ -126,6 +137,12 @@ public class TestInputPathHandler {
dfs.mkdirs(new Path(basePathTable4));
nonHoodiePaths.addAll(generatePartitions(dfs, basePathTable4));
initTableType(dfs.getConf(), basePathTable5, EMPTY_SNAPSHOT_TEST_NAME, HoodieTableType.COPY_ON_WRITE);
snapshotPaths.add(new Path(basePathTable5));
initTableType(dfs.getConf(), basePathTable6, EMPTY_INCREMENTAL_TEST_NAME, HoodieTableType.MERGE_ON_READ);
incrementalPaths.add(new Path(basePathTable6));
inputPaths.addAll(incrementalPaths);
inputPaths.addAll(snapshotPaths);
inputPaths.addAll(nonHoodiePaths);
@@ -133,6 +150,7 @@ public class TestInputPathHandler {
incrementalTables = new ArrayList<>();
incrementalTables.add(RAW_TRIPS_TEST_NAME);
incrementalTables.add(MODEL_TRIPS_TEST_NAME);
incrementalTables.add(EMPTY_INCREMENTAL_TEST_NAME);
}
static HoodieTableMetaClient initTableType(Configuration hadoopConf, String basePath,