From eb91e5ba7006ec9297fbc17658ee951afeb903f1 Mon Sep 17 00:00:00 2001 From: Sivabalan Narayanan Date: Wed, 3 Feb 2021 09:02:37 -0500 Subject: [PATCH] [HUDI-1523] Call mkdir(partition) only if not exists (#2501) --- .../src/main/java/org/apache/hudi/io/HoodieWriteHandle.java | 4 +++- .../src/main/java/org/apache/hudi/table/MarkerFiles.java | 4 +++- .../main/java/org/apache/hudi/io/HoodieRowCreateHandle.java | 4 +++- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieWriteHandle.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieWriteHandle.java index d2e538e2b..df0c2b6e2 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieWriteHandle.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieWriteHandle.java @@ -108,7 +108,9 @@ public abstract class HoodieWriteHandle public Path makeNewPath(String partitionPath) { Path path = FSUtils.getPartitionPath(config.getBasePath(), partitionPath); try { - fs.mkdirs(path); // create a new partition as needed. + if (!fs.exists(path)) { + fs.mkdirs(path); // create a new partition as needed. + } } catch (IOException e) { throw new HoodieIOException("Failed to make dir " + path, e); } diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/MarkerFiles.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/MarkerFiles.java index 9341a1d80..36112456f 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/MarkerFiles.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/MarkerFiles.java @@ -189,7 +189,9 @@ public class MarkerFiles implements Serializable { public Path create(String partitionPath, String dataFileName, IOType type) { Path path = FSUtils.getPartitionPath(markerDirPath, partitionPath); try { - fs.mkdirs(path); // create a new partition as needed. + if (!fs.exists(path)) { + fs.mkdirs(path); // create a new partition as needed. + } } catch (IOException e) { throw new HoodieIOException("Failed to make dir " + path, e); } diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/HoodieRowCreateHandle.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/HoodieRowCreateHandle.java index fa160c691..611e3c8b9 100644 --- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/HoodieRowCreateHandle.java +++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/HoodieRowCreateHandle.java @@ -172,7 +172,9 @@ public class HoodieRowCreateHandle implements Serializable { private Path makeNewPath(String partitionPath) { Path path = FSUtils.getPartitionPath(writeConfig.getBasePath(), partitionPath); try { - fs.mkdirs(path); // create a new partition as needed. + if (!fs.exists(path)) { + fs.mkdirs(path); // create a new partition as needed. + } } catch (IOException e) { throw new HoodieIOException("Failed to make dir " + path, e); }