lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-159] Precursor cleanup to reduce build warnings

Browse Source
This commit is contained in:
Vinoth Chandar
2019-08-26 17:41:00 -07:00
committed by Balaji Varadarajan
parent c265b4948f
commit 78e0721507
13 changed files with 110 additions and 87 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
docker/hoodie/hadoop/base/pom.xml
View File

@@ -32,15 +32,19 @@
<checkstyle.skip>true</checkstyle.skip> <checkstyle.skip>true</checkstyle.skip>
</properties> </properties>
<dependencies>
<dependency> <dependencyManagement>
<groupId>org.apache.hudi</groupId> <dependencies>
<artifactId>hudi-hadoop-docker</artifactId> <dependency>
<version>${project.version}</version> <groupId>org.apache.hudi</groupId>
<type>pom</type> <artifactId>hudi-hadoop-docker</artifactId>
<scope>import</scope> <version>${project.version}</version>
</dependency> <type>pom</type>
</dependencies> <scope>import</scope>
</dependency>
</dependencies>
</dependencyManagement>
<build> <build>
<finalName>hudi</finalName> <finalName>hudi</finalName>

20
docker/hoodie/hadoop/datanode/pom.xml
View File

@@ -32,15 +32,17 @@
<checkstyle.skip>true</checkstyle.skip> <checkstyle.skip>true</checkstyle.skip>
</properties> </properties>
<dependencies> <dependencyManagement>
<dependency> <dependencies>
<groupId>org.apache.hudi</groupId> <dependency>
<artifactId>hudi-hadoop-base-docker</artifactId> <groupId>org.apache.hudi</groupId>
<version>${project.version}</version> <artifactId>hudi-hadoop-base-docker</artifactId>
<type>pom</type> <version>${project.version}</version>
<scope>import</scope> <type>pom</type>
</dependency> <scope>import</scope>
</dependencies> </dependency>
</dependencies>
</dependencyManagement>
<build> <build>
<plugins> <plugins>

20
docker/hoodie/hadoop/historyserver/pom.xml
View File

@@ -32,15 +32,17 @@
<checkstyle.skip>true</checkstyle.skip> <checkstyle.skip>true</checkstyle.skip>
</properties> </properties>
<dependencies> <dependencyManagement>
<dependency> <dependencies>
<groupId>org.apache.hudi</groupId> <dependency>
<artifactId>hudi-hadoop-base-docker</artifactId> <groupId>org.apache.hudi</groupId>
<version>${project.version}</version> <artifactId>hudi-hadoop-base-docker</artifactId>
<type>pom</type> <version>${project.version}</version>
<scope>import</scope> <type>pom</type>
</dependency> <scope>import</scope>
</dependencies> </dependency>
</dependencies>
</dependencyManagement>
<build> <build>
<plugins> <plugins>

20
docker/hoodie/hadoop/hive_base/pom.xml
View File

@@ -32,15 +32,17 @@
<checkstyle.skip>true</checkstyle.skip> <checkstyle.skip>true</checkstyle.skip>
</properties> </properties>
<dependencies> <dependencyManagement>
<dependency> <dependencies>
<groupId>org.apache.hudi</groupId> <dependency>
<artifactId>hudi-hadoop-base-docker</artifactId> <groupId>org.apache.hudi</groupId>
<version>${project.version}</version> <artifactId>hudi-hadoop-base-docker</artifactId>
<type>pom</type> <version>${project.version}</version>
<scope>import</scope> <type>pom</type>
</dependency> <scope>import</scope>
</dependencies> </dependency>
</dependencies>
</dependencyManagement>
<build> <build>
<plugins> <plugins>

20
docker/hoodie/hadoop/namenode/pom.xml
View File

@@ -32,15 +32,17 @@
<checkstyle.skip>true</checkstyle.skip> <checkstyle.skip>true</checkstyle.skip>
</properties> </properties>
<dependencies> <dependencyManagement>
<dependency> <dependencies>
<groupId>org.apache.hudi</groupId> <dependency>
<artifactId>hudi-hadoop-base-docker</artifactId> <groupId>org.apache.hudi</groupId>
<version>${project.version}</version> <artifactId>hudi-hadoop-base-docker</artifactId>
<type>pom</type> <version>${project.version}</version>
<scope>import</scope> <type>pom</type>
</dependency> <scope>import</scope>
</dependencies> </dependency>
</dependencies>
</dependencyManagement>
<build> <build>
<plugins> <plugins>

20
docker/hoodie/hadoop/prestobase/pom.xml
View File

@@ -33,15 +33,17 @@
<checkstyle.skip>true</checkstyle.skip> <checkstyle.skip>true</checkstyle.skip>
</properties> </properties>
<dependencies> <dependencyManagement>
<dependency> <dependencies>
<groupId>org.apache.hudi</groupId> <dependency>
<artifactId>hudi-hadoop-base-docker</artifactId> <groupId>org.apache.hudi</groupId>
<version>${project.version}</version> <artifactId>hudi-hadoop-base-docker</artifactId>
<type>pom</type> <version>${project.version}</version>
<scope>import</scope> <type>pom</type>
</dependency> <scope>import</scope>
</dependencies> </dependency>
</dependencies>
</dependencyManagement>
<build> <build>
<plugins> <plugins>

20
docker/hoodie/hadoop/spark_base/pom.xml
View File

@@ -32,15 +32,17 @@
<checkstyle.skip>true</checkstyle.skip> <checkstyle.skip>true</checkstyle.skip>
</properties> </properties>
<dependencies> <dependencyManagement>
<dependency> <dependencies>
<groupId>org.apache.hudi</groupId> <dependency>
<artifactId>hudi-hadoop-hive-docker</artifactId> <groupId>org.apache.hudi</groupId>
<version>${project.version}</version> <artifactId>hudi-hadoop-hive-docker</artifactId>
<type>pom</type> <version>${project.version}</version>
<scope>import</scope> <type>pom</type>
</dependency> <scope>import</scope>
</dependencies> </dependency>
</dependencies>
</dependencyManagement>
<build> <build>
<plugins> <plugins>

20
docker/hoodie/hadoop/sparkadhoc/pom.xml
View File

@@ -32,15 +32,17 @@
<checkstyle.skip>true</checkstyle.skip> <checkstyle.skip>true</checkstyle.skip>
</properties> </properties>
<dependencies> <dependencyManagement>
<dependency> <dependencies>
<groupId>org.apache.hudi</groupId> <dependency>
<artifactId>hudi-hadoop-sparkbase-docker</artifactId> <groupId>org.apache.hudi</groupId>
<version>${project.version}</version> <artifactId>hudi-hadoop-sparkbase-docker</artifactId>
<type>pom</type> <version>${project.version}</version>
<scope>import</scope> <type>pom</type>
</dependency> <scope>import</scope>
</dependencies> </dependency>
</dependencies>
</dependencyManagement>
<build> <build>
<plugins> <plugins>

2
docker/hoodie/hadoop/sparkmaster/pom.xml
View File

@@ -32,6 +32,7 @@
<checkstyle.skip>true</checkstyle.skip> <checkstyle.skip>true</checkstyle.skip>
</properties> </properties>
<dependencyManagement>
<dependencies> <dependencies>
<dependency> <dependency>
<groupId>org.apache.hudi</groupId> <groupId>org.apache.hudi</groupId>
@@ -41,6 +42,7 @@
<scope>import</scope> <scope>import</scope>
</dependency> </dependency>
</dependencies> </dependencies>
</dependencyManagement>
<build> <build>
<plugins> <plugins>

20
docker/hoodie/hadoop/sparkworker/pom.xml
View File

@@ -32,15 +32,17 @@
<checkstyle.skip>true</checkstyle.skip> <checkstyle.skip>true</checkstyle.skip>
</properties> </properties>
<dependencies> <dependencyManagement>
<dependency> <dependencies>
<groupId>org.apache.hudi</groupId> <dependency>
<artifactId>hudi-hadoop-sparkbase-docker</artifactId> <groupId>org.apache.hudi</groupId>
<version>${project.version}</version> <artifactId>hudi-hadoop-sparkbase-docker</artifactId>
<type>pom</type> <version>${project.version}</version>
<scope>import</scope> <type>pom</type>
</dependency> <scope>import</scope>
</dependencies> </dependency>
</dependencies>
</dependencyManagement>
<build> <build>
<plugins> <plugins>

6
hudi-cli/src/main/scala/org/apache/hudi/cli/SparkHelpers.scala
View File

@@ -107,7 +107,9 @@ class SparkHelper(sqlContext: SQLContext, fs: FileSystem) {
* @param sqlContext * @param sqlContext
*/ */
def getKeyCount(file: String, sqlContext: org.apache.spark.sql.SQLContext) = { def getKeyCount(file: String, sqlContext: org.apache.spark.sql.SQLContext) = {
println(getRowKeyDF(file).collect().length) val keyCount = getRowKeyDF(file).collect().length
println(keyCount)
keyCount
} }
@@ -128,7 +130,7 @@ class SparkHelper(sqlContext: SQLContext, fs: FileSystem) {
.select(s"`${HoodieRecord.RECORD_KEY_METADATA_FIELD}`") .select(s"`${HoodieRecord.RECORD_KEY_METADATA_FIELD}`")
.collect().count(r => !bf.mightContain(r.getString(0))) .collect().count(r => !bf.mightContain(r.getString(0)))
val totalCount = getKeyCount(file, sqlContext) val totalCount = getKeyCount(file, sqlContext)
s"totalCount: ${totalCount}, foundCount: ${foundCount}" println(s"totalCount: $totalCount, foundCount: $foundCount")
totalCount == foundCount totalCount == foundCount
} }

2
hudi-integ-test/pom.xml
View File

@@ -56,7 +56,6 @@
<artifactId>hudi-hadoop-sparkworker-docker</artifactId> <artifactId>hudi-hadoop-sparkworker-docker</artifactId>
<version>${project.version}</version> <version>${project.version}</version>
<type>pom</type> <type>pom</type>
<scope>import</scope>
</dependency> </dependency>
<!-- Hoodie - Test --> <!-- Hoodie - Test -->
@@ -130,6 +129,7 @@
<plugin> <plugin>
<artifactId>exec-maven-plugin</artifactId> <artifactId>exec-maven-plugin</artifactId>
<groupId>org.codehaus.mojo</groupId> <groupId>org.codehaus.mojo</groupId>
<version>1.6.0</version>
<executions> <executions>
<execution><!-- setup HUDI_WS variable in docker compose env file --> <execution><!-- setup HUDI_WS variable in docker compose env file -->
<id>Setup HUDI_WS</id> <id>Setup HUDI_WS</id>

5
pom.xml
View File

@@ -159,6 +159,7 @@
<skipTests>false</skipTests> <skipTests>false</skipTests>
<skipITs>${skipTests}</skipITs> <skipITs>${skipTests}</skipITs>
<skipUTs>${skipTests}</skipUTs> <skipUTs>${skipTests}</skipUTs>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
</properties> </properties>
<scm> <scm>
@@ -253,7 +254,7 @@
<failOnWarning>false</failOnWarning> <failOnWarning>false</failOnWarning>
<sourceDirectory>${project.basedir}/src/main/scala</sourceDirectory> <sourceDirectory>${project.basedir}/src/main/scala</sourceDirectory>
<testSourceDirectory>${project.basedir}/src/test/scala</testSourceDirectory> <testSourceDirectory>${project.basedir}/src/test/scala</testSourceDirectory>
<configLocation>style/scalastyle-config.xml</configLocation> <configLocation>${maven.multiModuleProjectDirectory}/style/scalastyle-config.xml</configLocation>
<outputEncoding>UTF-8</outputEncoding> <outputEncoding>UTF-8</outputEncoding>
</configuration> </configuration>
<executions> <executions>
@@ -302,8 +303,6 @@
<version>${maven-surefire-plugin.version}</version> <version>${maven-surefire-plugin.version}</version>
<configuration> <configuration>
<skip>${skipUTs}</skip> <skip>${skipUTs}</skip>
<!-- Sets the VM argument line used when unit tests are run. -->
<argLine>${surefireArgLine}</argLine>
<systemPropertyVariables> <systemPropertyVariables>
<log4j.configuration> <log4j.configuration>
${surefire-log4j.file} ${surefire-log4j.file}