From 35111131c389704b2f5d4062c8ab73582f61ef13 Mon Sep 17 00:00:00 2001 From: Sivabalan Narayanan Date: Tue, 2 Nov 2021 00:16:45 -0400 Subject: [PATCH] [HUDI-2005] Fixing partition path creation in AbstractTableFileSystemView (#3769) --- .../table/action/commit/TestUpsertPartitioner.java | 3 ++- .../common/table/view/AbstractTableFileSystemView.java | 10 +++++++--- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/action/commit/TestUpsertPartitioner.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/action/commit/TestUpsertPartitioner.java index 3a125d230..7b5cc27d3 100644 --- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/action/commit/TestUpsertPartitioner.java +++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/action/commit/TestUpsertPartitioner.java @@ -21,6 +21,7 @@ package org.apache.hudi.table.action.commit; import org.apache.hudi.avro.model.HoodieClusteringPlan; import org.apache.hudi.avro.model.HoodieCompactionPlan; import org.apache.hudi.avro.model.HoodieRequestedReplaceMetadata; +import org.apache.hudi.common.config.HoodieMetadataConfig; import org.apache.hudi.common.model.HoodieCommitMetadata; import org.apache.hudi.common.model.HoodieRecord; import org.apache.hudi.common.model.HoodieRecordLocation; @@ -217,7 +218,7 @@ public class TestUpsertPartitioner extends HoodieClientTestBase { final String testPartitionPath = "2016/09/26"; int totalInsertNum = 2000; - HoodieWriteConfig config = makeHoodieClientConfigBuilder() + HoodieWriteConfig config = makeHoodieClientConfigBuilder().withMetadataConfig(HoodieMetadataConfig.newBuilder().enable(false).build()) .withCompactionConfig(HoodieCompactionConfig.newBuilder().compactionSmallFileSize(0) .insertSplitSize(totalInsertNum / 2).autoTuneInsertSplits(false).build()).build(); diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/view/AbstractTableFileSystemView.java b/hudi-common/src/main/java/org/apache/hudi/common/table/view/AbstractTableFileSystemView.java index 01122fdc5..eca3718f1 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/view/AbstractTableFileSystemView.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/view/AbstractTableFileSystemView.java @@ -285,9 +285,7 @@ public abstract class AbstractTableFileSystemView implements SyncableFileSystemV try { LOG.info("Building file system view for partition (" + partitionPathStr + ")"); - // Create the path if it does not exist already Path partitionPath = FSUtils.getPartitionPath(metaClient.getBasePath(), partitionPathStr); - FSUtils.createPathIfNotExists(metaClient.getFs(), partitionPath); long beginLsTs = System.currentTimeMillis(); FileStatus[] statuses = listPartition(partitionPath); long endLsTs = System.currentTimeMillis(); @@ -317,7 +315,13 @@ public abstract class AbstractTableFileSystemView implements SyncableFileSystemV * @throws IOException */ protected FileStatus[] listPartition(Path partitionPath) throws IOException { - return metaClient.getFs().listStatus(partitionPath); + // Create the path if it does not exist already + if (!metaClient.getFs().exists(partitionPath)) { + metaClient.getFs().mkdirs(partitionPath); + return new FileStatus[0]; + } else { + return metaClient.getFs().listStatus(partitionPath); + } } /**