From 43a0776c7c88a5f7beac6c8853db7e341810635a Mon Sep 17 00:00:00 2001
From: ZhangChaoMing <72908278+ZhangChaoming@users.noreply.github.com>
Date: Sun, 21 Feb 2021 11:54:04 +0800
Subject: [PATCH] [HUDI-1586] [Common Core] [Flink Integration] Reduce the
 coupling of hadoop. (#2540)

Co-authored-by: zhangchaoming <zhangchaoming@360.com>
---
 hudi-common/pom.xml                           |  2 +
 .../org/apache/hudi/common/fs/FSUtils.java    |  7 +--
 hudi-flink/pom.xml                            | 43 ++-----------------
 3 files changed, 8 insertions(+), 44 deletions(-)

diff --git a/hudi-common/pom.xml b/hudi-common/pom.xml
index 2981155a2..41588ca89 100644
--- a/hudi-common/pom.xml
+++ b/hudi-common/pom.xml
@@ -144,6 +144,7 @@
           <artifactId>*</artifactId>
         </exclusion>
       </exclusions>
+      <scope>provided</scope>
     </dependency>
     <dependency>
       <groupId>org.apache.hadoop</groupId>
@@ -154,6 +155,7 @@
     <dependency>
       <groupId>org.apache.hadoop</groupId>
       <artifactId>hadoop-hdfs</artifactId>
+      <scope>provided</scope>
     </dependency>
     <dependency>
       <groupId>org.apache.hadoop</groupId>
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/fs/FSUtils.java b/hudi-common/src/main/java/org/apache/hudi/common/fs/FSUtils.java
index a2b388982..341e3f5bc 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/fs/FSUtils.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/fs/FSUtils.java
@@ -80,9 +80,6 @@ public class FSUtils {
   private static final PathFilter ALLOW_ALL_FILTER = file -> true;
 
   public static Configuration prepareHadoopConf(Configuration conf) {
-    conf.set("fs.hdfs.impl", org.apache.hadoop.hdfs.DistributedFileSystem.class.getName());
-    conf.set("fs.file.impl", org.apache.hadoop.fs.LocalFileSystem.class.getName());
-
     // look for all properties, prefixed to be picked up
     for (Entry<String, String> prop : System.getenv().entrySet()) {
       if (prop.getKey().startsWith(HOODIE_ENV_PROPS_PREFIX)) {
@@ -607,8 +604,8 @@ public class FSUtils {
    * Helper to filter out paths under metadata folder when running fs.globStatus.
    * @param fs  File System
    * @param globPath Glob Path
-   * @return
-   * @throws IOException
+   * @return the file status list of globPath exclude the meta folder
+   * @throws IOException when having trouble listing the path
    */
   public static List<FileStatus> getGlobStatusExcludingMetaFolder(FileSystem fs, Path globPath) throws IOException {
     FileStatus[] statuses = fs.globStatus(globPath);
diff --git a/hudi-flink/pom.xml b/hudi-flink/pom.xml
index 2a0f39556..81a7849a5 100644
--- a/hudi-flink/pom.xml
+++ b/hudi-flink/pom.xml
@@ -153,39 +153,11 @@
       <scope>provided</scope>
     </dependency>
 
-    <!-- Hadoop -->
+    <!-- Parquet -->
     <dependency>
-      <groupId>org.apache.hadoop</groupId>
-      <artifactId>hadoop-common</artifactId>
-      <scope>compile</scope>
-      <exclusions>
-        <exclusion>
-          <groupId>org.slf4j</groupId>
-          <artifactId>slf4j-log4j12</artifactId>
-        </exclusion>
-      </exclusions>
-    </dependency>
-    <dependency>
-      <groupId>org.apache.hadoop</groupId>
-      <artifactId>hadoop-hdfs</artifactId>
-      <scope>compile</scope>
-      <exclusions>
-        <exclusion>
-          <groupId>org.slf4j</groupId>
-          <artifactId>slf4j-log4j12</artifactId>
-        </exclusion>
-      </exclusions>
-    </dependency>
-    <dependency>
-      <groupId>org.apache.hadoop</groupId>
-      <artifactId>hadoop-auth</artifactId>
-      <scope>compile</scope>
-      <exclusions>
-        <exclusion>
-          <groupId>org.slf4j</groupId>
-          <artifactId>slf4j-log4j12</artifactId>
-        </exclusion>
-      </exclusions>
+      <groupId>org.apache.parquet</groupId>
+      <artifactId>parquet-avro</artifactId>
+      <scope>test</scope>
     </dependency>
 
     <!-- Avro -->
@@ -197,13 +169,6 @@
       <scope>compile</scope>
     </dependency>
 
-    <!-- Parquet -->
-    <dependency>
-      <groupId>org.apache.parquet</groupId>
-      <artifactId>parquet-avro</artifactId>
-      <scope>compile</scope>
-    </dependency>
-
     <!-- Hadoop -->
     <dependency>
       <groupId>org.apache.hadoop</groupId>