diff --git a/README.md b/README.md index b8f38763b..2ee7556de 100644 --- a/README.md +++ b/README.md @@ -101,6 +101,12 @@ mvn clean package -DskipTests -Pspark2.4,scala-2.12 Starting from versions 0.11, Hudi no longer requires `spark-avro` to be specified using `--packages` +### Usage of hudi-utilities-slim-bundle + +Starting from versions 0.11, Hudi provides hudi-utilities-slim-bundle which excludes hudi-spark-datasource modules. +This new bundle is intended to be used with Hudi Spark bundle together, if using hudi-utilities-bundle solely +introduces problems for a specific Spark version. + ## Running Tests Unit tests can be run with maven profile `unit-tests`. diff --git a/packaging/hudi-utilities-slim-bundle/pom.xml b/packaging/hudi-utilities-slim-bundle/pom.xml new file mode 100644 index 000000000..848ece34e --- /dev/null +++ b/packaging/hudi-utilities-slim-bundle/pom.xml @@ -0,0 +1,477 @@ + + + + + hudi + org.apache.hudi + 0.11.0-SNAPSHOT + ../../pom.xml + + 4.0.0 + hudi-utilities-slim-bundle_${scala.binary.version} + jar + + + true + ${project.parent.basedir} + + + + + + org.apache.rat + apache-rat-plugin + + false + 0 + + NOTICE + DISCLAIMER + **/.* + **/*.json + **/*.log + **/*.sqltemplate + **/compose_env + **/*NOTICE* + **/*LICENSE* + **/dependency-reduced-pom.xml + **/test/resources/*.data + **/test/resources/*.commit + **/target/** + **/generated-sources/** + .github/** + **/*.sql + + + + + org.apache.maven.plugins + maven-shade-plugin + ${maven-shade-plugin.version} + + + package + + shade + + + ${shadeSources} + ${project.build.directory}/dependency-reduced-pom.xml + + + + + + true + + + META-INF/LICENSE + target/classes/META-INF/LICENSE + + + META-INF/services/org.apache.spark.sql.sources.DataSourceRegister + + + + + + org.apache.hudi:hudi-common + org.apache.hudi:hudi-client-common + org.apache.hudi:hudi-spark-client + org.apache.hudi:hudi-utilities_${scala.binary.version} + org.apache.hudi:hudi-hive-sync + org.apache.hudi:hudi-sync-common + org.apache.hudi:hudi-hadoop-mr + org.apache.hudi:hudi-timeline-service + org.apache.hudi:hudi-aws + + com.yammer.metrics:metrics-core + com.beust:jcommander + io.javalin:javalin + + org.eclipse.jetty:* + org.eclipse.jetty.websocket:* + org.jetbrains.kotlin:* + org.rocksdb:rocksdbjni + org.apache.httpcomponents:httpclient + org.apache.httpcomponents:httpcore + org.apache.httpcomponents:fluent-hc + org.antlr:stringtemplate + org.apache.parquet:parquet-avro + + com.github.davidmoten:guava-mini + com.github.davidmoten:hilbert-curve + com.twitter:bijection-avro_${scala.binary.version} + com.twitter:bijection-core_${scala.binary.version} + io.confluent:kafka-avro-serializer + io.confluent:kafka-schema-serializer + io.confluent:common-config + io.confluent:common-utils + io.confluent:kafka-schema-registry-client + io.dropwizard.metrics:metrics-core + io.dropwizard.metrics:metrics-graphite + io.dropwizard.metrics:metrics-jmx + io.prometheus:simpleclient + io.prometheus:simpleclient_httpserver + io.prometheus:simpleclient_dropwizard + io.prometheus:simpleclient_pushgateway + io.prometheus:simpleclient_common + com.yammer.metrics:metrics-core + org.apache.spark:spark-streaming-kafka-0-10_${scala.binary.version} + org.apache.spark:spark-token-provider-kafka-0-10_${scala.binary.version} + org.apache.kafka:kafka_${scala.binary.version} + com.101tec:zkclient + org.apache.kafka:kafka-clients + + org.apache.hive:hive-common + org.apache.hive:hive-service + org.apache.hive:hive-service-rpc + org.apache.hive:hive-metastore + org.apache.hive:hive-jdbc + + org.apache.hbase:hbase-client + org.apache.hbase:hbase-common + org.apache.hbase:hbase-hadoop-compat + org.apache.hbase:hbase-hadoop2-compat + org.apache.hbase:hbase-metrics + org.apache.hbase:hbase-metrics-api + org.apache.hbase:hbase-protocol-shaded + org.apache.hbase:hbase-server + org.apache.hbase.thirdparty:hbase-shaded-miscellaneous + org.apache.hbase.thirdparty:hbase-shaded-netty + org.apache.hbase.thirdparty:hbase-shaded-protobuf + org.apache.htrace:htrace-core4 + org.apache.curator:curator-framework + org.apache.curator:curator-client + org.apache.curator:curator-recipes + commons-codec:commons-codec + commons-io:commons-io + + + + + + org.apache.spark.sql.avro. + org.apache.hudi.org.apache.spark.sql.avro. + + + com.yammer.metrics. + org.apache.hudi.com.yammer.metrics. + + + com.beust.jcommander. + org.apache.hudi.com.beust.jcommander. + + + org.apache.hive.jdbc. + ${utilities.bundle.hive.shade.prefix}org.apache.hive.jdbc. + + + org.apache.commons.io. + org.apache.hudi.org.apache.commons.io. + + + org.apache.hadoop.hbase. + org.apache.hudi.org.apache.hadoop.hbase. + + org.apache.hadoop.hbase.KeyValue$KeyComparator + + + + org.apache.hbase. + org.apache.hudi.org.apache.hbase. + + + org.apache.htrace. + org.apache.hudi.org.apache.htrace. + + + org.apache.hadoop.hive.metastore. + ${utilities.bundle.hive.shade.prefix}org.apache.hadoop.hive.metastore. + + + org.apache.hive.common. + ${utilities.bundle.hive.shade.prefix}org.apache.hive.common. + + + org.apache.hadoop.hive.common. + ${utilities.bundle.hive.shade.prefix}org.apache.hadoop.hive.common. + + + org.apache.hadoop.hive.conf. + ${utilities.bundle.hive.shade.prefix}org.apache.hadoop.hive.conf. + + + org.apache.hive.service. + ${utilities.bundle.hive.shade.prefix}org.apache.hive.service. + + + org.apache.hadoop.hive.service. + ${utilities.bundle.hive.shade.prefix}org.apache.hadoop.hive.service. + + + com.codahale.metrics. + org.apache.hudi.com.codahale.metrics. + + + org.apache.commons.codec. + org.apache.hudi.org.apache.commons.codec. + + + org.eclipse.jetty. + org.apache.hudi.org.eclipse.jetty. + + + + org.apache.hadoop.metrics2.MetricHistogram + org.apache.hudi.org.apache.hadoop.metrics2.MetricHistogram + + + + org.apache.hadoop.metrics2.MetricsExecutor + org.apache.hudi.org.apache.hadoop.metrics2.MetricsExecutor + + + + org.apache.hadoop.metrics2.impl.JmxCacheBuster + org.apache.hudi.org.apache.hadoop.metrics2.impl.JmxCacheBuster + + + org.apache.hadoop.metrics2.lib.DefaultMetricsSystemHelper + org.apache.hudi.org.apache.hadoop.metrics2.lib.DefaultMetricsSystemHelper + + + + org.apache.hadoop.metrics2.lib.DynamicMetricsRegistry + org.apache.hudi.org.apache.hadoop.metrics2.lib.DynamicMetricsRegistry + + + + org.apache.hadoop.metrics2.lib.MetricsExecutorImpl + org.apache.hudi.org.apache.hadoop.metrics2.lib.MetricsExecutorImpl + + + + org.apache.hadoop.metrics2.lib.MutableFastCounter + org.apache.hudi.org.apache.hadoop.metrics2.lib.MutableFastCounter + + + + org.apache.hadoop.metrics2.lib.MutableHistogram + org.apache.hudi.org.apache.hadoop.metrics2.lib.MutableHistogram + + + + org.apache.hadoop.metrics2.lib.MutableRangeHistogram + org.apache.hudi.org.apache.hadoop.metrics2.lib.MutableRangeHistogram + + + + org.apache.hadoop.metrics2.lib.MutableSizeHistogram + org.apache.hudi.org.apache.hadoop.metrics2.lib.MutableSizeHistogram + + + + org.apache.hadoop.metrics2.lib.MutableTimeHistogram + org.apache.hudi.org.apache.hadoop.metrics2.lib.MutableTimeHistogram + + + + org.apache.hadoop.metrics2.util.MetricQuantile + org.apache.hudi.org.apache.hadoop.metrics2.util.MetricQuantile + + + + org.apache.hadoop.metrics2.util.MetricSampleQuantiles + org.apache.hudi.org.apache.hadoop.metrics2.util.MetricSampleQuantiles + + + + + + *:* + + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + META-INF/services/javax.* + **/*.proto + hbase-webapps/** + + + + + + + + + + + + src/main/resources + + + src/test/resources + + + + + + + + org.apache.hudi + hudi-common + ${project.version} + + + org.apache.hudi + hudi-client-common + ${project.version} + + + org.apache.hudi + hudi-spark-client + ${project.version} + + + org.apache.hudi + hudi-hive-sync + ${project.version} + + + javax.servlet + servlet-api + + + + + org.apache.hudi + hudi-spark-common_${scala.binary.version} + ${project.version} + provided + + + org.apache.hudi + hudi-spark_${scala.binary.version} + ${project.version} + provided + + + org.apache.hudi + ${hudi.spark.module}_${scala.binary.version} + ${project.version} + provided + + + org.apache.hudi + ${hudi.spark.common.module} + ${project.version} + provided + + + org.apache.hudi + hudi-utilities_${scala.binary.version} + ${project.version} + + + + + org.apache.parquet + parquet-avro + compile + + + + + ${hive.groupid} + hive-service + ${hive.version} + ${utilities.bundle.hive.scope} + + + + ${hive.groupid} + hive-service-rpc + ${hive.version} + ${utilities.bundle.hive.scope} + + + + ${hive.groupid} + hive-jdbc + ${hive.version} + ${utilities.bundle.hive.scope} + + + + ${hive.groupid} + hive-metastore + ${hive.version} + ${utilities.bundle.hive.scope} + + + + ${hive.groupid} + hive-common + ${hive.version} + ${utilities.bundle.hive.scope} + + + + org.apache.htrace + htrace-core + ${htrace.version} + compile + + + + + org.apache.curator + curator-framework + ${zk-curator.version} + + + + org.apache.curator + curator-client + ${zk-curator.version} + + + + org.apache.curator + curator-recipes + ${zk-curator.version} + + + + + + utilities-bundle-shade-hive + + compile + org.apache.hudi. + + + + + diff --git a/packaging/hudi-utilities-slim-bundle/src/main/java/org/apache/hudi/utilities/bundle/Main.java b/packaging/hudi-utilities-slim-bundle/src/main/java/org/apache/hudi/utilities/bundle/Main.java new file mode 100644 index 000000000..a4e2dcb34 --- /dev/null +++ b/packaging/hudi-utilities-slim-bundle/src/main/java/org/apache/hudi/utilities/bundle/Main.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.utilities.bundle; + +import org.apache.hudi.common.util.ReflectionUtils; + +/** + * A simple main class to dump all classes loaded in current classpath + * + * This is a workaround for generating sources and javadoc jars for packaging modules. The maven plugins for generating + * javadoc and sources plugins do not generate corresponding jars if there are no source files. + * + * This class does not have anything to do with Hudi but is there to keep mvn javadocs/source plugin happy. + */ +public class Main { + + public static void main(String[] args) { + ReflectionUtils.getTopLevelClassesInClasspath(Main.class).forEach(System.out::println); + } +} diff --git a/pom.xml b/pom.xml index d81525f97..d65586172 100644 --- a/pom.xml +++ b/pom.xml @@ -50,6 +50,7 @@ packaging/hudi-spark-bundle packaging/hudi-presto-bundle packaging/hudi-utilities-bundle + packaging/hudi-utilities-slim-bundle packaging/hudi-timeline-server-bundle packaging/hudi-trino-bundle hudi-examples