lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-2005] Fixing partition path creation in AbstractTableFileSystemView (#3769)

Browse Source
This commit is contained in:
Sivabalan Narayanan
2021-11-02 00:16:45 -04:00
committed by GitHub
parent 5302b9a4ef
commit 35111131c3
2 changed files with 9 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/action/commit/TestUpsertPartitioner.java
View File

@@ -21,6 +21,7 @@ package org.apache.hudi.table.action.commit;
import org.apache.hudi.avro.model.HoodieClusteringPlan; import org.apache.hudi.avro.model.HoodieClusteringPlan;
import org.apache.hudi.avro.model.HoodieCompactionPlan; import org.apache.hudi.avro.model.HoodieCompactionPlan;
import org.apache.hudi.avro.model.HoodieRequestedReplaceMetadata; import org.apache.hudi.avro.model.HoodieRequestedReplaceMetadata;
import org.apache.hudi.common.config.HoodieMetadataConfig;
import org.apache.hudi.common.model.HoodieCommitMetadata; import org.apache.hudi.common.model.HoodieCommitMetadata;
import org.apache.hudi.common.model.HoodieRecord; import org.apache.hudi.common.model.HoodieRecord;
import org.apache.hudi.common.model.HoodieRecordLocation; import org.apache.hudi.common.model.HoodieRecordLocation;
@@ -217,7 +218,7 @@ public class TestUpsertPartitioner extends HoodieClientTestBase {
final String testPartitionPath = "2016/09/26"; final String testPartitionPath = "2016/09/26";
int totalInsertNum = 2000; int totalInsertNum = 2000;
HoodieWriteConfig config = makeHoodieClientConfigBuilder() HoodieWriteConfig config = makeHoodieClientConfigBuilder().withMetadataConfig(HoodieMetadataConfig.newBuilder().enable(false).build())
.withCompactionConfig(HoodieCompactionConfig.newBuilder().compactionSmallFileSize(0) .withCompactionConfig(HoodieCompactionConfig.newBuilder().compactionSmallFileSize(0)
.insertSplitSize(totalInsertNum / 2).autoTuneInsertSplits(false).build()).build(); .insertSplitSize(totalInsertNum / 2).autoTuneInsertSplits(false).build()).build();

10
hudi-common/src/main/java/org/apache/hudi/common/table/view/AbstractTableFileSystemView.java
View File

@@ -285,9 +285,7 @@ public abstract class AbstractTableFileSystemView implements SyncableFileSystemV
try { try {
LOG.info("Building file system view for partition (" + partitionPathStr + ")"); LOG.info("Building file system view for partition (" + partitionPathStr + ")");
// Create the path if it does not exist already
Path partitionPath = FSUtils.getPartitionPath(metaClient.getBasePath(), partitionPathStr); Path partitionPath = FSUtils.getPartitionPath(metaClient.getBasePath(), partitionPathStr);
FSUtils.createPathIfNotExists(metaClient.getFs(), partitionPath);
long beginLsTs = System.currentTimeMillis(); long beginLsTs = System.currentTimeMillis();
FileStatus[] statuses = listPartition(partitionPath); FileStatus[] statuses = listPartition(partitionPath);
long endLsTs = System.currentTimeMillis(); long endLsTs = System.currentTimeMillis();
@@ -317,7 +315,13 @@ public abstract class AbstractTableFileSystemView implements SyncableFileSystemV
* @throws IOException * @throws IOException
*/ */
protected FileStatus[] listPartition(Path partitionPath) throws IOException { protected FileStatus[] listPartition(Path partitionPath) throws IOException {
return metaClient.getFs().listStatus(partitionPath); // Create the path if it does not exist already
if (!metaClient.getFs().exists(partitionPath)) {
metaClient.getFs().mkdirs(partitionPath);
return new FileStatus[0];
} else {
return metaClient.getFs().listStatus(partitionPath);
}
} }
/** /**