lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-3643] Fix hive count exception when the table is empty and the path depth is less than 3 (#5051)

Browse Source
This commit is contained in:
董可伦
2022-04-07 19:21:03 +08:00
committed by GitHub
parent 9d744bb35c
commit 6a8396420c
5 changed files with 63 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/utils/HoodieHiveUtils.java
View File

@@ -71,7 +71,6 @@ public class HoodieHiveUtils {
public static final String DEFAULT_SCAN_MODE = SNAPSHOT_SCAN_MODE;
public static final int DEFAULT_MAX_COMMITS = 1;
public static final int MAX_COMMIT_ALL = -1;
public static final int DEFAULT_LEVELS_TO_BASEPATH = 3;
public static final Pattern HOODIE_CONSUME_MODE_PATTERN_STRING = Pattern.compile("hoodie\\.(.*)\\.consume\\.mode");
public static final String GLOBALLY_CONSISTENT_READ_TIMESTAMP = "last_replication_timestamp";

18
hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/utils/HoodieInputFormatUtils.java
View File

@@ -46,6 +46,7 @@ import org.apache.hudi.common.table.view.TableFileSystemView;
import org.apache.hudi.common.util.Option;
import org.apache.hudi.common.util.StringUtils;
import org.apache.hudi.exception.HoodieIOException;
import org.apache.hudi.exception.TableNotFoundException;
import org.apache.hudi.hadoop.FileStatusWithBootstrapBaseFile;
import org.apache.hudi.hadoop.HoodieHFileInputFormat;
import org.apache.hudi.hadoop.HoodieParquetInputFormat;
@@ -68,6 +69,7 @@ import java.util.stream.Collectors;
import static org.apache.hudi.common.config.HoodieMetadataConfig.DEFAULT_METADATA_ENABLE_FOR_READERS;
import static org.apache.hudi.common.config.HoodieMetadataConfig.ENABLE;
import static org.apache.hudi.common.table.HoodieTableMetaClient.METAFOLDER_NAME;
public class HoodieInputFormatUtils {
@@ -324,14 +326,24 @@ public class HoodieInputFormatUtils {
* Extract HoodieTableMetaClient from a partition path (not base path)
*/
public static HoodieTableMetaClient getTableMetaClientForBasePathUnchecked(Configuration conf, Path partitionPath) throws IOException {
Path baseDir = partitionPath;
FileSystem fs = partitionPath.getFileSystem(conf);
int levels = HoodieHiveUtils.DEFAULT_LEVELS_TO_BASEPATH;
if (HoodiePartitionMetadata.hasPartitionMetadata(fs, partitionPath)) {
HoodiePartitionMetadata metadata = new HoodiePartitionMetadata(fs, partitionPath);
metadata.readFromFS();
levels = metadata.getPartitionDepth();
int levels = metadata.getPartitionDepth();
baseDir = HoodieHiveUtils.getNthParent(partitionPath, levels);
} else {
for (int i = 0; i < partitionPath.depth(); i++) {
if (fs.exists(new Path(baseDir, METAFOLDER_NAME))) {
break;
} else if (i == partitionPath.depth() - 1) {
throw new TableNotFoundException(partitionPath.toString());
} else {
baseDir = baseDir.getParent();
}
}
}
Path baseDir = HoodieHiveUtils.getNthParent(partitionPath, levels);
LOG.info("Reading hoodie metadata from path " + baseDir.toString());
return HoodieTableMetaClient.builder().setConf(fs.getConf()).setBasePath(baseDir.toString()).build();
}