lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-1586] [Common Core] [Flink Integration] Reduce the coupling of hadoop. (#2540)

Browse Source 
Co-authored-by: zhangchaoming <zhangchaoming@360.com>
This commit is contained in:
ZhangChaoMing
2021-02-21 11:54:04 +08:00
committed by GitHub
parent ffcfb58bac
commit 43a0776c7c
3 changed files with 8 additions and 44 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
hudi-common/pom.xml
View File

@@ -144,6 +144,7 @@
<artifactId>*</artifactId> <artifactId>*</artifactId>
</exclusion> </exclusion>
</exclusions> </exclusions>
<scope>provided</scope>
</dependency> </dependency>
<dependency> <dependency>
<groupId>org.apache.hadoop</groupId> <groupId>org.apache.hadoop</groupId>
@@ -154,6 +155,7 @@
<dependency> <dependency>
<groupId>org.apache.hadoop</groupId> <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdfs</artifactId> <artifactId>hadoop-hdfs</artifactId>
<scope>provided</scope>
</dependency> </dependency>
<dependency> <dependency>
<groupId>org.apache.hadoop</groupId> <groupId>org.apache.hadoop</groupId>

7
hudi-common/src/main/java/org/apache/hudi/common/fs/FSUtils.java
View File

@@ -80,9 +80,6 @@ public class FSUtils {
private static final PathFilter ALLOW_ALL_FILTER = file -> true; private static final PathFilter ALLOW_ALL_FILTER = file -> true;
public static Configuration prepareHadoopConf(Configuration conf) { public static Configuration prepareHadoopConf(Configuration conf) {
conf.set("fs.hdfs.impl", org.apache.hadoop.hdfs.DistributedFileSystem.class.getName());
conf.set("fs.file.impl", org.apache.hadoop.fs.LocalFileSystem.class.getName());
// look for all properties, prefixed to be picked up // look for all properties, prefixed to be picked up
for (Entry<String, String> prop : System.getenv().entrySet()) { for (Entry<String, String> prop : System.getenv().entrySet()) {
if (prop.getKey().startsWith(HOODIE_ENV_PROPS_PREFIX)) { if (prop.getKey().startsWith(HOODIE_ENV_PROPS_PREFIX)) {
@@ -607,8 +604,8 @@ public class FSUtils {
* Helper to filter out paths under metadata folder when running fs.globStatus. * Helper to filter out paths under metadata folder when running fs.globStatus.
* @param fs File System * @param fs File System
* @param globPath Glob Path * @param globPath Glob Path
* @return * @return the file status list of globPath exclude the meta folder
* @throws IOException * @throws IOException when having trouble listing the path
*/ */
public static List<FileStatus> getGlobStatusExcludingMetaFolder(FileSystem fs, Path globPath) throws IOException { public static List<FileStatus> getGlobStatusExcludingMetaFolder(FileSystem fs, Path globPath) throws IOException {
FileStatus[] statuses = fs.globStatus(globPath); FileStatus[] statuses = fs.globStatus(globPath);

43
hudi-flink/pom.xml
View File

@@ -153,39 +153,11 @@
<scope>provided</scope> <scope>provided</scope>
</dependency> </dependency>
<!-- Hadoop --> <!-- Parquet -->
<dependency> <dependency>
<groupId>org.apache.hadoop</groupId> <groupId>org.apache.parquet</groupId>
<artifactId>hadoop-common</artifactId> <artifactId>parquet-avro</artifactId>
<scope>compile</scope> <scope>test</scope>
<exclusions>
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdfs</artifactId>
<scope>compile</scope>
<exclusions>
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-auth</artifactId>
<scope>compile</scope>
<exclusions>
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
</exclusion>
</exclusions>
</dependency> </dependency>
<!-- Avro --> <!-- Avro -->
@@ -197,13 +169,6 @@
<scope>compile</scope> <scope>compile</scope>
</dependency> </dependency>
<!-- Parquet -->
<dependency>
<groupId>org.apache.parquet</groupId>
<artifactId>parquet-avro</artifactId>
<scope>compile</scope>
</dependency>
<!-- Hadoop --> <!-- Hadoop -->
<dependency> <dependency>
<groupId>org.apache.hadoop</groupId> <groupId>org.apache.hadoop</groupId>