de37774e12
Currently, all Hudi Relations bear performance gap relative to Spark's HadoopFsRelation and the reason to that is SchemaPruning optimization rule (pruning nested schemas) that is unfortunately predicated on usage of HadoopFsRelation, meaning that it's not applied in cases when any other relation is used. This change is porting this rule to Hudi relations (MOR, Incremental, etc) by the virtue of leveraging HoodieSparkSessionExtensions mechanism injecting modified version of the original SchemaPruning rule that is adopted to work w/ Hudi's custom relations. - Added customOptimizerRules to HoodieAnalysis - Added NestedSchemaPrunning Spark's Optimizer rule - Handle Spark's Optimizer pruned data schema (to effectively prune nested schemas) - Enable HoodieClientTestHarness to inject HoodieSparkSessionExtensions - Injecting Spark Session extensions for TestMORDataSource, TestCOWDataSource - Disabled fallback to HadoopFsRelation
558 lines
17 KiB
XML
558 lines
17 KiB
XML
<?xml version="1.0" encoding="UTF-8"?>
|
|
<!--
|
|
Licensed to the Apache Software Foundation (ASF) under one or more
|
|
contributor license agreements. See the NOTICE file distributed with
|
|
this work for additional information regarding copyright ownership.
|
|
The ASF licenses this file to You under the Apache License, Version 2.0
|
|
(the "License"); you may not use this file except in compliance with
|
|
the License. You may obtain a copy of the License at
|
|
|
|
http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
Unless required by applicable law or agreed to in writing, software
|
|
distributed under the License is distributed on an "AS IS" BASIS,
|
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
See the License for the specific language governing permissions and
|
|
limitations under the License.
|
|
-->
|
|
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
|
<parent>
|
|
<artifactId>hudi-spark-datasource</artifactId>
|
|
<groupId>org.apache.hudi</groupId>
|
|
<version>0.12.0-SNAPSHOT</version>
|
|
</parent>
|
|
<modelVersion>4.0.0</modelVersion>
|
|
|
|
<artifactId>hudi-spark_${scala.binary.version}</artifactId>
|
|
<version>0.12.0-SNAPSHOT</version>
|
|
|
|
<name>hudi-spark_${scala.binary.version}</name>
|
|
<packaging>jar</packaging>
|
|
|
|
<properties>
|
|
<main.basedir>${project.parent.parent.basedir}</main.basedir>
|
|
</properties>
|
|
|
|
<build>
|
|
<resources>
|
|
<resource>
|
|
<directory>src/main/resources</directory>
|
|
</resource>
|
|
</resources>
|
|
<pluginManagement>
|
|
<plugins>
|
|
<plugin>
|
|
<groupId>net.alchim31.maven</groupId>
|
|
<artifactId>scala-maven-plugin</artifactId>
|
|
<version>${scala-maven-plugin.version}</version>
|
|
<configuration>
|
|
<args>
|
|
<arg>-nobootcp</arg>
|
|
<arg>-target:jvm-1.8</arg>
|
|
</args>
|
|
<checkMultipleScalaVersions>false</checkMultipleScalaVersions>
|
|
</configuration>
|
|
</plugin>
|
|
<plugin>
|
|
<groupId>org.apache.maven.plugins</groupId>
|
|
<artifactId>maven-compiler-plugin</artifactId>
|
|
</plugin>
|
|
</plugins>
|
|
</pluginManagement>
|
|
|
|
<plugins>
|
|
<plugin>
|
|
<groupId>org.apache.maven.plugins</groupId>
|
|
<artifactId>maven-dependency-plugin</artifactId>
|
|
<executions>
|
|
<execution>
|
|
<id>copy-dependencies</id>
|
|
<phase>prepare-package</phase>
|
|
<goals>
|
|
<goal>copy-dependencies</goal>
|
|
</goals>
|
|
<configuration>
|
|
<outputDirectory>${project.build.directory}/lib</outputDirectory>
|
|
<overWriteReleases>true</overWriteReleases>
|
|
<overWriteSnapshots>true</overWriteSnapshots>
|
|
<overWriteIfNewer>true</overWriteIfNewer>
|
|
</configuration>
|
|
</execution>
|
|
</executions>
|
|
</plugin>
|
|
<plugin>
|
|
<groupId>net.alchim31.maven</groupId>
|
|
<artifactId>scala-maven-plugin</artifactId>
|
|
<executions>
|
|
<execution>
|
|
<id>scala-compile-first</id>
|
|
<phase>process-resources</phase>
|
|
<goals>
|
|
<goal>add-source</goal>
|
|
<goal>compile</goal>
|
|
</goals>
|
|
</execution>
|
|
<execution>
|
|
<id>scala-test-compile</id>
|
|
<phase>process-test-resources</phase>
|
|
<goals>
|
|
<goal>testCompile</goal>
|
|
</goals>
|
|
</execution>
|
|
</executions>
|
|
</plugin>
|
|
<plugin>
|
|
<groupId>org.apache.maven.plugins</groupId>
|
|
<artifactId>maven-compiler-plugin</artifactId>
|
|
<executions>
|
|
<execution>
|
|
<phase>compile</phase>
|
|
<goals>
|
|
<goal>compile</goal>
|
|
</goals>
|
|
</execution>
|
|
</executions>
|
|
</plugin>
|
|
<plugin>
|
|
<groupId>org.apache.maven.plugins</groupId>
|
|
<artifactId>maven-jar-plugin</artifactId>
|
|
<executions>
|
|
<execution>
|
|
<goals>
|
|
<goal>test-jar</goal>
|
|
</goals>
|
|
<phase>test-compile</phase>
|
|
</execution>
|
|
</executions>
|
|
<configuration>
|
|
<skip>false</skip>
|
|
</configuration>
|
|
</plugin>
|
|
<plugin>
|
|
<groupId>org.apache.rat</groupId>
|
|
<artifactId>apache-rat-plugin</artifactId>
|
|
</plugin>
|
|
<plugin>
|
|
<groupId>org.scalatest</groupId>
|
|
<artifactId>scalatest-maven-plugin</artifactId>
|
|
<version>1.0</version>
|
|
<configuration>
|
|
<skipTests>${skipUTs}</skipTests>
|
|
<reportsDirectory>${project.build.directory}/surefire-reports</reportsDirectory>
|
|
<junitxml>.</junitxml>
|
|
<filereports>TestSuite.txt</filereports>
|
|
</configuration>
|
|
<executions>
|
|
<execution>
|
|
<id>test</id>
|
|
<goals>
|
|
<goal>test</goal>
|
|
</goals>
|
|
</execution>
|
|
</executions>
|
|
</plugin>
|
|
<plugin>
|
|
<groupId>org.scalastyle</groupId>
|
|
<artifactId>scalastyle-maven-plugin</artifactId>
|
|
</plugin>
|
|
<plugin>
|
|
<groupId>org.jacoco</groupId>
|
|
<artifactId>jacoco-maven-plugin</artifactId>
|
|
</plugin>
|
|
<plugin>
|
|
<groupId>org.antlr</groupId>
|
|
<artifactId>antlr4-maven-plugin</artifactId>
|
|
<version>${antlr.version}</version>
|
|
<executions>
|
|
<execution>
|
|
<goals>
|
|
<goal>antlr4</goal>
|
|
</goals>
|
|
</execution>
|
|
</executions>
|
|
<configuration>
|
|
<visitor>true</visitor>
|
|
<listener>true</listener>
|
|
<sourceDirectory>../hudi-spark/src/main/antlr4/</sourceDirectory>
|
|
</configuration>
|
|
</plugin>
|
|
</plugins>
|
|
</build>
|
|
|
|
<dependencies>
|
|
<!-- Scala -->
|
|
<dependency>
|
|
<groupId>org.scala-lang</groupId>
|
|
<artifactId>scala-library</artifactId>
|
|
<version>${scala.version}</version>
|
|
</dependency>
|
|
|
|
<!-- Hoodie -->
|
|
<dependency>
|
|
<groupId>org.apache.hudi</groupId>
|
|
<artifactId>hudi-client-common</artifactId>
|
|
<version>${project.version}</version>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.apache.hudi</groupId>
|
|
<artifactId>hudi-spark-client</artifactId>
|
|
<version>${project.version}</version>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.apache.hudi</groupId>
|
|
<artifactId>hudi-common</artifactId>
|
|
<version>${project.version}</version>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.apache.hudi</groupId>
|
|
<artifactId>hudi-hadoop-mr</artifactId>
|
|
<version>${project.version}</version>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.apache.hudi</groupId>
|
|
<artifactId>hudi-hive-sync</artifactId>
|
|
<version>${project.version}</version>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.apache.hudi</groupId>
|
|
<artifactId>hudi-sync-common</artifactId>
|
|
<version>${project.version}</version>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.apache.hudi</groupId>
|
|
<artifactId>hudi-spark-common_${scala.binary.version}</artifactId>
|
|
<version>${project.version}</version>
|
|
<exclusions>
|
|
<exclusion>
|
|
<groupId>org.apache.curator</groupId>
|
|
<artifactId>*</artifactId>
|
|
</exclusion>
|
|
</exclusions>
|
|
</dependency>
|
|
|
|
<dependency>
|
|
<groupId>org.apache.hudi</groupId>
|
|
<artifactId>${hudi.spark.module}_${scala.binary.version}</artifactId>
|
|
<version>${project.version}</version>
|
|
<exclusions>
|
|
<exclusion>
|
|
<groupId>org.apache.hudi</groupId>
|
|
<artifactId>*</artifactId>
|
|
</exclusion>
|
|
</exclusions>
|
|
</dependency>
|
|
|
|
<dependency>
|
|
<groupId>org.apache.hudi</groupId>
|
|
<artifactId>${hudi.spark.common.module}</artifactId>
|
|
<version>${project.version}</version>
|
|
<exclusions>
|
|
<exclusion>
|
|
<groupId>org.apache.hudi</groupId>
|
|
<artifactId>*</artifactId>
|
|
</exclusion>
|
|
</exclusions>
|
|
</dependency>
|
|
|
|
<!-- Logging -->
|
|
<dependency>
|
|
<groupId>org.apache.logging.log4j</groupId>
|
|
<artifactId>log4j-api</artifactId>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.apache.logging.log4j</groupId>
|
|
<artifactId>log4j-core</artifactId>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.apache.logging.log4j</groupId>
|
|
<artifactId>log4j-1.2-api</artifactId>
|
|
</dependency>
|
|
|
|
<!-- Fasterxml -->
|
|
<dependency>
|
|
<groupId>com.fasterxml.jackson.core</groupId>
|
|
<artifactId>jackson-annotations</artifactId>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>com.fasterxml.jackson.module</groupId>
|
|
<artifactId>jackson-module-scala_${scala.binary.version}</artifactId>
|
|
</dependency>
|
|
|
|
<!-- Avro -->
|
|
<dependency>
|
|
<groupId>org.apache.avro</groupId>
|
|
<artifactId>avro</artifactId>
|
|
<exclusions>
|
|
<exclusion>
|
|
<!-- this version to conflict to spark-core_2.12 -->
|
|
<groupId>com.thoughtworks.paranamer</groupId>
|
|
<artifactId>paranamer</artifactId>
|
|
</exclusion>
|
|
</exclusions>
|
|
</dependency>
|
|
|
|
<!-- Parquet -->
|
|
<dependency>
|
|
<groupId>org.apache.parquet</groupId>
|
|
<artifactId>parquet-avro</artifactId>
|
|
</dependency>
|
|
|
|
<!-- Spark -->
|
|
<dependency>
|
|
<groupId>org.apache.spark</groupId>
|
|
<artifactId>spark-core_${scala.binary.version}</artifactId>
|
|
<exclusions>
|
|
<exclusion>
|
|
<groupId>javax.servlet</groupId>
|
|
<artifactId>*</artifactId>
|
|
</exclusion>
|
|
</exclusions>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.apache.spark</groupId>
|
|
<artifactId>spark-sql_${scala.binary.version}</artifactId>
|
|
</dependency>
|
|
|
|
<dependency>
|
|
<groupId>org.apache.spark</groupId>
|
|
<artifactId>spark-hive_${scala.binary.version}</artifactId>
|
|
</dependency>
|
|
|
|
<dependency>
|
|
<groupId>org.apache.spark</groupId>
|
|
<artifactId>spark-sql_${scala.binary.version}</artifactId>
|
|
<classifier>tests</classifier>
|
|
<scope>test</scope>
|
|
</dependency>
|
|
|
|
<dependency>
|
|
<groupId>org.apache.spark</groupId>
|
|
<artifactId>spark-core_${scala.binary.version}</artifactId>
|
|
<classifier>tests</classifier>
|
|
<scope>test</scope>
|
|
</dependency>
|
|
|
|
<dependency>
|
|
<groupId>org.apache.spark</groupId>
|
|
<artifactId>spark-catalyst_${scala.binary.version}</artifactId>
|
|
<classifier>tests</classifier>
|
|
<scope>test</scope>
|
|
</dependency>
|
|
|
|
<!-- Hadoop -->
|
|
<dependency>
|
|
<groupId>org.apache.hadoop</groupId>
|
|
<artifactId>hadoop-client</artifactId>
|
|
<exclusions>
|
|
<exclusion>
|
|
<groupId>javax.servlet</groupId>
|
|
<artifactId>*</artifactId>
|
|
</exclusion>
|
|
</exclusions>
|
|
<scope>provided</scope>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.apache.hadoop</groupId>
|
|
<artifactId>hadoop-common</artifactId>
|
|
<exclusions>
|
|
<exclusion>
|
|
<groupId>javax.servlet</groupId>
|
|
<artifactId>*</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>javax.servlet.jsp</groupId>
|
|
<artifactId>*</artifactId>
|
|
</exclusion>
|
|
</exclusions>
|
|
<scope>provided</scope>
|
|
</dependency>
|
|
|
|
<!-- Hive -->
|
|
<dependency>
|
|
<groupId>${hive.groupid}</groupId>
|
|
<artifactId>hive-exec</artifactId>
|
|
<version>${hive.version}</version>
|
|
<classifier>${hive.exec.classifier}</classifier>
|
|
<exclusions>
|
|
<exclusion>
|
|
<groupId>javax.mail</groupId>
|
|
<artifactId>mail</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>org.eclipse.jetty.aggregate</groupId>
|
|
<artifactId>*</artifactId>
|
|
</exclusion>
|
|
</exclusions>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>${hive.groupid}</groupId>
|
|
<artifactId>hive-jdbc</artifactId>
|
|
<version>${hive.version}</version>
|
|
<exclusions>
|
|
<exclusion>
|
|
<groupId>javax.servlet</groupId>
|
|
<artifactId>*</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>javax.servlet.jsp</groupId>
|
|
<artifactId>*</artifactId>
|
|
</exclusion>
|
|
</exclusions>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>${hive.groupid}</groupId>
|
|
<artifactId>hive-metastore</artifactId>
|
|
<version>${hive.version}</version>
|
|
<exclusions>
|
|
<exclusion>
|
|
<groupId>javax.servlet</groupId>
|
|
<artifactId>*</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>javax.servlet.jsp</groupId>
|
|
<artifactId>*</artifactId>
|
|
</exclusion>
|
|
</exclusions>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>${hive.groupid}</groupId>
|
|
<artifactId>hive-common</artifactId>
|
|
<version>${hive.version}</version>
|
|
<exclusions>
|
|
<exclusion>
|
|
<groupId>org.eclipse.jetty.orbit</groupId>
|
|
<artifactId>javax.servlet</artifactId>
|
|
</exclusion>
|
|
</exclusions>
|
|
</dependency>
|
|
|
|
<dependency>
|
|
<groupId>org.apache.curator</groupId>
|
|
<artifactId>curator-framework</artifactId>
|
|
<version>${zk-curator.version}</version>
|
|
</dependency>
|
|
|
|
<dependency>
|
|
<groupId>org.apache.curator</groupId>
|
|
<artifactId>curator-client</artifactId>
|
|
<version>${zk-curator.version}</version>
|
|
</dependency>
|
|
|
|
<dependency>
|
|
<groupId>org.apache.curator</groupId>
|
|
<artifactId>curator-recipes</artifactId>
|
|
<version>${zk-curator.version}</version>
|
|
</dependency>
|
|
|
|
<!-- Hoodie - Test -->
|
|
<dependency>
|
|
<groupId>org.apache.hudi</groupId>
|
|
<artifactId>hudi-client-common</artifactId>
|
|
<version>${project.version}</version>
|
|
<classifier>tests</classifier>
|
|
<type>test-jar</type>
|
|
<scope>test</scope>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.apache.hudi</groupId>
|
|
<artifactId>hudi-spark-client</artifactId>
|
|
<version>${project.version}</version>
|
|
<classifier>tests</classifier>
|
|
<type>test-jar</type>
|
|
<scope>test</scope>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.apache.hudi</groupId>
|
|
<artifactId>hudi-common</artifactId>
|
|
<version>${project.version}</version>
|
|
<classifier>tests</classifier>
|
|
<type>test-jar</type>
|
|
<scope>test</scope>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.apache.hudi</groupId>
|
|
<artifactId>hudi-java-client</artifactId>
|
|
<version>${project.version}</version>
|
|
<scope>test</scope>
|
|
</dependency>
|
|
|
|
<dependency>
|
|
<groupId>org.scalatest</groupId>
|
|
<artifactId>scalatest_${scala.binary.version}</artifactId>
|
|
<version>${scalatest.version}</version>
|
|
<scope>test</scope>
|
|
</dependency>
|
|
|
|
<dependency>
|
|
<groupId>org.junit.jupiter</groupId>
|
|
<artifactId>junit-jupiter-api</artifactId>
|
|
<scope>test</scope>
|
|
</dependency>
|
|
|
|
<dependency>
|
|
<groupId>org.junit.jupiter</groupId>
|
|
<artifactId>junit-jupiter-engine</artifactId>
|
|
<scope>test</scope>
|
|
</dependency>
|
|
|
|
<dependency>
|
|
<groupId>org.junit.vintage</groupId>
|
|
<artifactId>junit-vintage-engine</artifactId>
|
|
<scope>test</scope>
|
|
</dependency>
|
|
|
|
<dependency>
|
|
<groupId>org.junit.jupiter</groupId>
|
|
<artifactId>junit-jupiter-params</artifactId>
|
|
<scope>test</scope>
|
|
</dependency>
|
|
|
|
<dependency>
|
|
<groupId>org.mockito</groupId>
|
|
<artifactId>mockito-junit-jupiter</artifactId>
|
|
<scope>test</scope>
|
|
</dependency>
|
|
|
|
<dependency>
|
|
<groupId>org.junit.platform</groupId>
|
|
<artifactId>junit-platform-runner</artifactId>
|
|
<scope>test</scope>
|
|
</dependency>
|
|
|
|
<dependency>
|
|
<groupId>org.junit.platform</groupId>
|
|
<artifactId>junit-platform-suite-api</artifactId>
|
|
<scope>test</scope>
|
|
</dependency>
|
|
|
|
<dependency>
|
|
<groupId>org.slf4j</groupId>
|
|
<artifactId>slf4j-api</artifactId>
|
|
<version>${slf4j.version}</version>
|
|
<scope>test</scope>
|
|
</dependency>
|
|
|
|
<dependency>
|
|
<groupId>org.apache.hadoop</groupId>
|
|
<artifactId>hadoop-hdfs</artifactId>
|
|
<classifier>tests</classifier>
|
|
<scope>test</scope>
|
|
<!-- Need these exclusions to make sure JavaSparkContext can be setup. https://issues.apache.org/jira/browse/SPARK-1693 -->
|
|
<exclusions>
|
|
<exclusion>
|
|
<groupId>org.mortbay.jetty</groupId>
|
|
<artifactId>*</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>javax.servlet.jsp</groupId>
|
|
<artifactId>*</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>javax.servlet</groupId>
|
|
<artifactId>*</artifactId>
|
|
</exclusion>
|
|
</exclusions>
|
|
</dependency>
|
|
</dependencies>
|
|
</project>
|