diff --git a/docker/demo/setup_demo_container.sh b/docker/demo/setup_demo_container.sh index 067e5fdf7..717472c37 100755 --- a/docker/demo/setup_demo_container.sh +++ b/docker/demo/setup_demo_container.sh @@ -21,4 +21,4 @@ cp /var/hoodie/ws/docker/demo/config/spark-defaults.conf $SPARK_CONF_DIR/. hadoop fs -mkdir -p /var/demo/ hadoop fs -mkdir -p /tmp/spark-events hadoop fs -copyFromLocal -f /var/hoodie/ws/docker/demo/config /var/demo/. -chmod +x /var/hoodie/ws/hoodie-hive/run_sync_tool.sh +chmod +x /var/hoodie/ws/hudi-hive/run_sync_tool.sh diff --git a/docker/hoodie/hadoop/base/pom.xml b/docker/hoodie/hadoop/base/pom.xml index b06061051..f02918433 100644 --- a/docker/hoodie/hadoop/base/pom.xml +++ b/docker/hoodie/hadoop/base/pom.xml @@ -17,13 +17,13 @@ --> - hoodie-hadoop-docker - com.uber.hoodie - 0.4.8-SNAPSHOT + hudi-hadoop-docker + org.apache.hudi + 0.5.0-SNAPSHOT 4.0.0 pom - hoodie-hadoop-base-docker + hudi-hadoop-base-docker Base Docker Image with Hoodie @@ -34,8 +34,8 @@ - com.uber.hoodie - hoodie-hadoop-docker + org.apache.hudi + hudi-hadoop-docker ${project.version} pom import @@ -43,7 +43,7 @@ - hoodie + hudi diff --git a/docker/hoodie/hadoop/datanode/pom.xml b/docker/hoodie/hadoop/datanode/pom.xml index 207b4af87..0df35d670 100644 --- a/docker/hoodie/hadoop/datanode/pom.xml +++ b/docker/hoodie/hadoop/datanode/pom.xml @@ -17,13 +17,13 @@ --> - hoodie-hadoop-docker - com.uber.hoodie - 0.4.8-SNAPSHOT + hudi-hadoop-docker + org.apache.hudi + 0.5.0-SNAPSHOT 4.0.0 pom - hoodie-hadoop-datanode-docker + hudi-hadoop-datanode-docker Base Docker Image with Hoodie @@ -34,8 +34,8 @@ - com.uber.hoodie - hoodie-hadoop-base-docker + org.apache.hudi + hudi-hadoop-base-docker ${project.version} pom import diff --git a/docker/hoodie/hadoop/historyserver/pom.xml b/docker/hoodie/hadoop/historyserver/pom.xml index 7cfd84a64..bf7a9946d 100644 --- a/docker/hoodie/hadoop/historyserver/pom.xml +++ b/docker/hoodie/hadoop/historyserver/pom.xml @@ -17,13 +17,13 @@ --> - hoodie-hadoop-docker - com.uber.hoodie - 0.4.8-SNAPSHOT + hudi-hadoop-docker + org.apache.hudi + 0.5.0-SNAPSHOT 4.0.0 pom - hoodie-hadoop-history-docker + hudi-hadoop-history-docker Base Docker Image with Hoodie @@ -34,8 +34,8 @@ - com.uber.hoodie - hoodie-hadoop-base-docker + org.apache.hudi + hudi-hadoop-base-docker ${project.version} pom import diff --git a/docker/hoodie/hadoop/hive_base/pom.xml b/docker/hoodie/hadoop/hive_base/pom.xml index eade6c69c..8a934c8a1 100644 --- a/docker/hoodie/hadoop/hive_base/pom.xml +++ b/docker/hoodie/hadoop/hive_base/pom.xml @@ -17,13 +17,13 @@ --> - hoodie-hadoop-docker - com.uber.hoodie - 0.4.8-SNAPSHOT + hudi-hadoop-docker + org.apache.hudi + 0.5.0-SNAPSHOT 4.0.0 pom - hoodie-hadoop-hive-docker + hudi-hadoop-hive-docker Base Docker Image with Hoodie @@ -34,8 +34,8 @@ - com.uber.hoodie - hoodie-hadoop-base-docker + org.apache.hudi + hudi-hadoop-base-docker ${project.version} pom import @@ -52,11 +52,11 @@ package - - - + + + diff --git a/docker/hoodie/hadoop/namenode/pom.xml b/docker/hoodie/hadoop/namenode/pom.xml index 4a98d8787..8f01c27fb 100644 --- a/docker/hoodie/hadoop/namenode/pom.xml +++ b/docker/hoodie/hadoop/namenode/pom.xml @@ -17,13 +17,13 @@ --> - hoodie-hadoop-docker - com.uber.hoodie - 0.4.8-SNAPSHOT + hudi-hadoop-docker + org.apache.hudi + 0.5.0-SNAPSHOT 4.0.0 pom - hoodie-hadoop-namenode-docker + hudi-hadoop-namenode-docker Base Docker Image with Hoodie @@ -34,8 +34,8 @@ - com.uber.hoodie - hoodie-hadoop-base-docker + org.apache.hudi + hudi-hadoop-base-docker ${project.version} pom import diff --git a/docker/hoodie/hadoop/pom.xml b/docker/hoodie/hadoop/pom.xml index 2ef18e305..7191fd407 100644 --- a/docker/hoodie/hadoop/pom.xml +++ b/docker/hoodie/hadoop/pom.xml @@ -17,14 +17,14 @@ --> - hoodie - com.uber.hoodie - 0.4.8-SNAPSHOT + hudi + org.apache.hudi + 0.5.0-SNAPSHOT ../../../pom.xml 4.0.0 - hoodie-hadoop-docker + hudi-hadoop-docker pom base @@ -40,8 +40,8 @@ - com.uber.hoodie - hoodie-spark-bundle + org.apache.hudi + hudi-spark-bundle ${project.version} diff --git a/docker/hoodie/hadoop/spark_base/pom.xml b/docker/hoodie/hadoop/spark_base/pom.xml index 8f9affea9..3f17d75e7 100644 --- a/docker/hoodie/hadoop/spark_base/pom.xml +++ b/docker/hoodie/hadoop/spark_base/pom.xml @@ -17,13 +17,13 @@ --> - hoodie-hadoop-docker - com.uber.hoodie - 0.4.8-SNAPSHOT + hudi-hadoop-docker + org.apache.hudi + 0.5.0-SNAPSHOT 4.0.0 pom - hoodie-hadoop-sparkbase-docker + hudi-hadoop-sparkbase-docker Base Docker Image with Hoodie @@ -34,8 +34,8 @@ - com.uber.hoodie - hoodie-hadoop-hive-docker + org.apache.hudi + hudi-hadoop-hive-docker ${project.version} pom import diff --git a/docker/hoodie/hadoop/sparkadhoc/pom.xml b/docker/hoodie/hadoop/sparkadhoc/pom.xml index 4bd4ffac3..e16142859 100644 --- a/docker/hoodie/hadoop/sparkadhoc/pom.xml +++ b/docker/hoodie/hadoop/sparkadhoc/pom.xml @@ -17,13 +17,13 @@ --> - hoodie-hadoop-docker - com.uber.hoodie - 0.4.8-SNAPSHOT + hudi-hadoop-docker + org.apache.hudi + 0.5.0-SNAPSHOT 4.0.0 pom - hoodie-hadoop-sparkadhoc-docker + hudi-hadoop-sparkadhoc-docker Base Docker Image with Hoodie @@ -34,8 +34,8 @@ - com.uber.hoodie - hoodie-hadoop-sparkbase-docker + org.apache.hudi + hudi-hadoop-sparkbase-docker ${project.version} pom import diff --git a/docker/hoodie/hadoop/sparkmaster/pom.xml b/docker/hoodie/hadoop/sparkmaster/pom.xml index 82cc0e2b1..a81e50bfe 100644 --- a/docker/hoodie/hadoop/sparkmaster/pom.xml +++ b/docker/hoodie/hadoop/sparkmaster/pom.xml @@ -17,13 +17,13 @@ --> - hoodie-hadoop-docker - com.uber.hoodie - 0.4.8-SNAPSHOT + hudi-hadoop-docker + org.apache.hudi + 0.5.0-SNAPSHOT 4.0.0 pom - hoodie-hadoop-sparkmaster-docker + hudi-hadoop-sparkmaster-docker Base Docker Image with Hoodie @@ -34,8 +34,8 @@ - com.uber.hoodie - hoodie-hadoop-sparkbase-docker + org.apache.hudi + hudi-hadoop-sparkbase-docker ${project.version} pom import diff --git a/docker/hoodie/hadoop/sparkworker/pom.xml b/docker/hoodie/hadoop/sparkworker/pom.xml index b2767f856..991e7c4ab 100644 --- a/docker/hoodie/hadoop/sparkworker/pom.xml +++ b/docker/hoodie/hadoop/sparkworker/pom.xml @@ -17,13 +17,13 @@ --> - hoodie-hadoop-docker - com.uber.hoodie - 0.4.8-SNAPSHOT + hudi-hadoop-docker + org.apache.hudi + 0.5.0-SNAPSHOT 4.0.0 pom - hoodie-hadoop-sparkworker-docker + hudi-hadoop-sparkworker-docker Base Docker Image with Hoodie @@ -34,8 +34,8 @@ - com.uber.hoodie - hoodie-hadoop-sparkbase-docker + org.apache.hudi + hudi-hadoop-sparkbase-docker ${project.version} pom import diff --git a/hoodie-utilities/src/test/java/com/uber/hoodie/utilities/SchedulerConfGeneratorTest.java b/hoodie-utilities/src/test/java/com/uber/hoodie/utilities/SchedulerConfGeneratorTest.java deleted file mode 100644 index 2e88d897e..000000000 --- a/hoodie-utilities/src/test/java/com/uber/hoodie/utilities/SchedulerConfGeneratorTest.java +++ /dev/null @@ -1,36 +0,0 @@ -package com.uber.hoodie.utilities; - -import static com.uber.hoodie.utilities.deltastreamer.SchedulerConfGenerator.SPARK_SCHEDULER_ALLOCATION_FILE_KEY; -import static com.uber.hoodie.utilities.deltastreamer.SchedulerConfGenerator.SPARK_SCHEDULER_MODE_KEY; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; - -import com.uber.hoodie.common.model.HoodieTableType; -import com.uber.hoodie.utilities.deltastreamer.HoodieDeltaStreamer; -import com.uber.hoodie.utilities.deltastreamer.SchedulerConfGenerator; -import java.util.Map; -import org.junit.Test; - -public class SchedulerConfGeneratorTest { - - @Test - public void testGenerateSparkSchedulingConf() throws Exception { - HoodieDeltaStreamer.Config cfg = new HoodieDeltaStreamer.Config(); - Map configs = SchedulerConfGenerator.getSparkSchedulingConfigs(cfg); - assertNull("spark.scheduler.mode not set", configs.get(SPARK_SCHEDULER_ALLOCATION_FILE_KEY)); - - System.setProperty(SPARK_SCHEDULER_MODE_KEY, "FAIR"); - cfg.continuousMode = false; - configs = SchedulerConfGenerator.getSparkSchedulingConfigs(cfg); - assertNull("continuousMode is false", configs.get(SPARK_SCHEDULER_ALLOCATION_FILE_KEY)); - - cfg.continuousMode = true; - cfg.storageType = HoodieTableType.COPY_ON_WRITE.name(); - configs = SchedulerConfGenerator.getSparkSchedulingConfigs(cfg); - assertNull("storageType is not MERGE_ON_READ", configs.get(SPARK_SCHEDULER_ALLOCATION_FILE_KEY)); - - cfg.storageType = HoodieTableType.MERGE_ON_READ.name(); - configs = SchedulerConfGenerator.getSparkSchedulingConfigs(cfg); - assertNotNull("all satisfies", configs.get(SPARK_SCHEDULER_ALLOCATION_FILE_KEY)); - } -} diff --git a/hoodie-cli/hoodie-cli.sh b/hudi-cli/hudi-cli.sh similarity index 50% rename from hoodie-cli/hoodie-cli.sh rename to hudi-cli/hudi-cli.sh index b29568bdc..188c16331 100755 --- a/hoodie-cli/hoodie-cli.sh +++ b/hudi-cli/hudi-cli.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -HOODIE_JAR=`ls $DIR/target/hoodie-cli-*-SNAPSHOT.jar | grep -v source | grep -v javadoc` +HUDI_JAR=`ls $DIR/target/hudi-cli-*-SNAPSHOT.jar | grep -v source | grep -v javadoc` if [ -z "$HADOOP_CONF_DIR" ]; then echo "setting hadoop conf dir" HADOOP_CONF_DIR="/etc/hadoop/conf" @@ -13,4 +13,5 @@ fi if [ -z "$CLIENT_JAR" ]; then echo "client jar location not set" fi -java -cp ${HADOOP_CONF_DIR}:${SPARK_CONF_DIR}:$DIR/target/lib/*:$HOODIE_JAR:${CLIENT_JAR} -DSPARK_CONF_DIR=${SPARK_CONF_DIR} -DHADOOP_CONF_DIR=${HADOOP_CONF_DIR} org.springframework.shell.Bootstrap +echo "java -cp ${HADOOP_CONF_DIR}:${SPARK_CONF_DIR}:$DIR/target/lib/*:$HUDI_JAR:${CLIENT_JAR} -DSPARK_CONF_DIR=${SPARK_CONF_DIR} -DHADOOP_CONF_DIR=${HADOOP_CONF_DIR} org.springframework.shell.Bootstrap" +java -cp ${HADOOP_CONF_DIR}:${SPARK_CONF_DIR}:$DIR/target/lib/*:$HUDI_JAR:${CLIENT_JAR} -DSPARK_CONF_DIR=${SPARK_CONF_DIR} -DHADOOP_CONF_DIR=${HADOOP_CONF_DIR} org.springframework.shell.Bootstrap diff --git a/hoodie-cli/lib/dnl/utils/textutils/0.3.3/textutils-0.3.3.jar b/hudi-cli/lib/dnl/utils/textutils/0.3.3/textutils-0.3.3.jar similarity index 100% rename from hoodie-cli/lib/dnl/utils/textutils/0.3.3/textutils-0.3.3.jar rename to hudi-cli/lib/dnl/utils/textutils/0.3.3/textutils-0.3.3.jar diff --git a/hoodie-cli/pom.xml b/hudi-cli/pom.xml similarity index 92% rename from hoodie-cli/pom.xml rename to hudi-cli/pom.xml index 6701b1aef..0ba33dcba 100644 --- a/hoodie-cli/pom.xml +++ b/hudi-cli/pom.xml @@ -17,13 +17,13 @@ --> - hoodie - com.uber.hoodie - 0.4.8-SNAPSHOT + hudi + org.apache.hudi + 0.5.0-SNAPSHOT 4.0.0 - hoodie-cli + hudi-cli jar @@ -172,14 +172,14 @@ - com.uber.hoodie - hoodie-hive + org.apache.hudi + hudi-hive ${project.version} - com.uber.hoodie - hoodie-client + org.apache.hudi + hudi-client ${project.version} @@ -194,8 +194,8 @@ - com.uber.hoodie - hoodie-common + org.apache.hudi + hudi-common ${project.version} @@ -215,8 +215,8 @@ 2.9.6 - com.uber.hoodie - hoodie-utilities + org.apache.hudi + hudi-utilities ${project.version} diff --git a/hoodie-cli/src/main/java/com/uber/hoodie/cli/HoodieCLI.java b/hudi-cli/src/main/java/org/apache/hudi/cli/HoodieCLI.java similarity index 92% rename from hoodie-cli/src/main/java/com/uber/hoodie/cli/HoodieCLI.java rename to hudi-cli/src/main/java/org/apache/hudi/cli/HoodieCLI.java index ba9c023c8..71c61a7f5 100644 --- a/hoodie-cli/src/main/java/com/uber/hoodie/cli/HoodieCLI.java +++ b/hudi-cli/src/main/java/org/apache/hudi/cli/HoodieCLI.java @@ -16,14 +16,14 @@ * limitations under the License. */ -package com.uber.hoodie.cli; +package org.apache.hudi.cli; -import com.uber.hoodie.common.table.HoodieTableMetaClient; -import com.uber.hoodie.common.util.ConsistencyGuardConfig; -import com.uber.hoodie.common.util.FSUtils; import java.io.IOException; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; +import org.apache.hudi.common.table.HoodieTableMetaClient; +import org.apache.hudi.common.util.ConsistencyGuardConfig; +import org.apache.hudi.common.util.FSUtils; public class HoodieCLI { diff --git a/hoodie-cli/src/main/java/com/uber/hoodie/cli/HoodieHistoryFileNameProvider.java b/hudi-cli/src/main/java/org/apache/hudi/cli/HoodieHistoryFileNameProvider.java similarity index 97% rename from hoodie-cli/src/main/java/com/uber/hoodie/cli/HoodieHistoryFileNameProvider.java rename to hudi-cli/src/main/java/org/apache/hudi/cli/HoodieHistoryFileNameProvider.java index 99f2d52ee..eeb2ff2a9 100644 --- a/hoodie-cli/src/main/java/com/uber/hoodie/cli/HoodieHistoryFileNameProvider.java +++ b/hudi-cli/src/main/java/org/apache/hudi/cli/HoodieHistoryFileNameProvider.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package com.uber.hoodie.cli; +package org.apache.hudi.cli; import org.springframework.core.Ordered; import org.springframework.core.annotation.Order; diff --git a/hoodie-cli/src/main/java/com/uber/hoodie/cli/HoodiePrintHelper.java b/hudi-cli/src/main/java/org/apache/hudi/cli/HoodiePrintHelper.java similarity index 97% rename from hoodie-cli/src/main/java/com/uber/hoodie/cli/HoodiePrintHelper.java rename to hudi-cli/src/main/java/org/apache/hudi/cli/HoodiePrintHelper.java index ebd097911..a5b64953a 100644 --- a/hoodie-cli/src/main/java/com/uber/hoodie/cli/HoodiePrintHelper.java +++ b/hudi-cli/src/main/java/org/apache/hudi/cli/HoodiePrintHelper.java @@ -16,13 +16,13 @@ * limitations under the License. */ -package com.uber.hoodie.cli; +package org.apache.hudi.cli; import com.jakewharton.fliptables.FlipTable; -import com.uber.hoodie.common.util.Option; import java.util.List; import java.util.Map; import java.util.function.Function; +import org.apache.hudi.common.util.Option; /** * Helper class to render table for hoodie-cli diff --git a/hoodie-cli/src/main/java/com/uber/hoodie/cli/HoodiePrompt.java b/hudi-cli/src/main/java/org/apache/hudi/cli/HoodiePrompt.java similarity index 83% rename from hoodie-cli/src/main/java/com/uber/hoodie/cli/HoodiePrompt.java rename to hudi-cli/src/main/java/org/apache/hudi/cli/HoodiePrompt.java index 066ce07b3..4ad0ad2d6 100644 --- a/hoodie-cli/src/main/java/com/uber/hoodie/cli/HoodiePrompt.java +++ b/hudi-cli/src/main/java/org/apache/hudi/cli/HoodiePrompt.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package com.uber.hoodie.cli; +package org.apache.hudi.cli; import org.springframework.core.Ordered; import org.springframework.core.annotation.Order; @@ -33,16 +33,16 @@ public class HoodiePrompt extends DefaultPromptProvider { String tableName = HoodieCLI.tableMetadata.getTableConfig().getTableName(); switch (HoodieCLI.state) { case INIT: - return "hoodie->"; + return "hudi->"; case DATASET: - return "hoodie:" + tableName + "->"; + return "hudi:" + tableName + "->"; case SYNC: - return "hoodie:" + tableName + " <==> " + HoodieCLI.syncTableMetadata.getTableConfig().getTableName() + "->"; + return "hudi:" + tableName + " <==> " + HoodieCLI.syncTableMetadata.getTableConfig().getTableName() + "->"; default: - return "hoodie:" + tableName + "->"; + return "hudi:" + tableName + "->"; } } - return "hoodie->"; + return "hudi->"; } @Override diff --git a/hoodie-cli/src/main/java/com/uber/hoodie/cli/HoodieSplashScreen.java b/hudi-cli/src/main/java/org/apache/hudi/cli/HoodieSplashScreen.java similarity index 78% rename from hoodie-cli/src/main/java/com/uber/hoodie/cli/HoodieSplashScreen.java rename to hudi-cli/src/main/java/org/apache/hudi/cli/HoodieSplashScreen.java index f58e8cefa..085680db9 100644 --- a/hoodie-cli/src/main/java/com/uber/hoodie/cli/HoodieSplashScreen.java +++ b/hudi-cli/src/main/java/org/apache/hudi/cli/HoodieSplashScreen.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package com.uber.hoodie.cli; +package org.apache.hudi.cli; import org.springframework.core.Ordered; import org.springframework.core.annotation.Order; @@ -28,14 +28,18 @@ import org.springframework.stereotype.Component; @Order(Ordered.HIGHEST_PRECEDENCE) public class HoodieSplashScreen extends DefaultBannerProvider { + static { + System.out.println("HoodieSplashScreen loaded"); + } + private static String screen = "============================================" + OsUtils.LINE_SEPARATOR + "* *" + OsUtils.LINE_SEPARATOR - + "* _ _ _ _ *" + OsUtils.LINE_SEPARATOR - + "* | | | | | (_) *" + OsUtils.LINE_SEPARATOR - + "* | |__| | ___ ___ __| |_ ___ *" + OsUtils.LINE_SEPARATOR - + "* | __ |/ _ \\ / _ \\ / _` | |/ _ \\ *" + OsUtils.LINE_SEPARATOR - + "* | | | | (_) | (_) | (_| | | __/ *" + OsUtils.LINE_SEPARATOR - + "* |_| |_|\\___/ \\___/ \\__,_|_|\\___| *" + OsUtils.LINE_SEPARATOR + + "* _ _ _ _ *" + OsUtils.LINE_SEPARATOR + + "* | | | | | | (_) *" + OsUtils.LINE_SEPARATOR + + "* | |__| | __| | - *" + OsUtils.LINE_SEPARATOR + + "* | __ || | / _` | || *" + OsUtils.LINE_SEPARATOR + + "* | | | || || (_| | || *" + OsUtils.LINE_SEPARATOR + + "* |_| |_|\\___/ \\____/ || *" + OsUtils.LINE_SEPARATOR + "* *" + OsUtils.LINE_SEPARATOR + "============================================" + OsUtils.LINE_SEPARATOR; diff --git a/hoodie-cli/src/main/java/com/uber/hoodie/cli/Main.java b/hudi-cli/src/main/java/org/apache/hudi/cli/Main.java similarity index 91% rename from hoodie-cli/src/main/java/com/uber/hoodie/cli/Main.java rename to hudi-cli/src/main/java/org/apache/hudi/cli/Main.java index bf8fc3847..b960336f7 100644 --- a/hoodie-cli/src/main/java/com/uber/hoodie/cli/Main.java +++ b/hudi-cli/src/main/java/org/apache/hudi/cli/Main.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package com.uber.hoodie.cli; +package org.apache.hudi.cli; import java.io.IOException; import org.springframework.shell.Bootstrap; @@ -27,6 +27,8 @@ public class Main { * Main class that delegates to Spring Shell's Bootstrap class in order to simplify debugging inside an IDE */ public static void main(String[] args) throws IOException { + System.out.println("Main called"); + new HoodieSplashScreen(); Bootstrap.main(args); } } diff --git a/hoodie-cli/src/main/java/com/uber/hoodie/cli/Table.java b/hudi-cli/src/main/java/org/apache/hudi/cli/Table.java similarity index 98% rename from hoodie-cli/src/main/java/com/uber/hoodie/cli/Table.java rename to hudi-cli/src/main/java/org/apache/hudi/cli/Table.java index c4cd43a97..79ee7cc92 100644 --- a/hoodie-cli/src/main/java/com/uber/hoodie/cli/Table.java +++ b/hudi-cli/src/main/java/org/apache/hudi/cli/Table.java @@ -16,9 +16,8 @@ * limitations under the License. */ -package com.uber.hoodie.cli; +package org.apache.hudi.cli; -import com.uber.hoodie.common.util.Option; import java.util.ArrayList; import java.util.Arrays; import java.util.Comparator; @@ -29,6 +28,7 @@ import java.util.function.Consumer; import java.util.function.Function; import java.util.stream.Collectors; import java.util.stream.IntStream; +import org.apache.hudi.common.util.Option; /** * Table to be rendered. This class takes care of ordering diff --git a/hoodie-cli/src/main/java/com/uber/hoodie/cli/TableHeader.java b/hudi-cli/src/main/java/org/apache/hudi/cli/TableHeader.java similarity index 98% rename from hoodie-cli/src/main/java/com/uber/hoodie/cli/TableHeader.java rename to hudi-cli/src/main/java/org/apache/hudi/cli/TableHeader.java index 4c73a562d..0472b0f73 100644 --- a/hoodie-cli/src/main/java/com/uber/hoodie/cli/TableHeader.java +++ b/hudi-cli/src/main/java/org/apache/hudi/cli/TableHeader.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package com.uber.hoodie.cli; +package org.apache.hudi.cli; import java.util.ArrayList; import java.util.List; diff --git a/hoodie-cli/src/main/java/com/uber/hoodie/cli/commands/ArchivedCommitsCommand.java b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/ArchivedCommitsCommand.java similarity index 93% rename from hoodie-cli/src/main/java/com/uber/hoodie/cli/commands/ArchivedCommitsCommand.java rename to hudi-cli/src/main/java/org/apache/hudi/cli/commands/ArchivedCommitsCommand.java index dce6a5e1e..c599c98c8 100644 --- a/hoodie-cli/src/main/java/com/uber/hoodie/cli/commands/ArchivedCommitsCommand.java +++ b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/ArchivedCommitsCommand.java @@ -16,18 +16,8 @@ * limitations under the License. */ -package com.uber.hoodie.cli.commands; +package org.apache.hudi.cli.commands; -import com.uber.hoodie.avro.model.HoodieArchivedMetaEntry; -import com.uber.hoodie.avro.model.HoodieCommitMetadata; -import com.uber.hoodie.cli.HoodieCLI; -import com.uber.hoodie.cli.HoodiePrintHelper; -import com.uber.hoodie.cli.TableHeader; -import com.uber.hoodie.common.model.HoodieLogFile; -import com.uber.hoodie.common.table.HoodieTimeline; -import com.uber.hoodie.common.table.log.HoodieLogFormat; -import com.uber.hoodie.common.table.log.block.HoodieAvroDataBlock; -import com.uber.hoodie.common.util.FSUtils; import java.io.IOException; import java.util.ArrayList; import java.util.HashMap; @@ -38,6 +28,17 @@ import org.apache.avro.generic.IndexedRecord; import org.apache.avro.specific.SpecificData; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.Path; +import org.apache.hudi.avro.model.HoodieArchivedMetaEntry; +import org.apache.hudi.avro.model.HoodieCommitMetadata; +import org.apache.hudi.cli.HoodieCLI; +import org.apache.hudi.cli.HoodiePrintHelper; +import org.apache.hudi.cli.TableHeader; +import org.apache.hudi.common.model.HoodieLogFile; +import org.apache.hudi.common.table.HoodieTimeline; +import org.apache.hudi.common.table.log.HoodieLogFormat; +import org.apache.hudi.common.table.log.HoodieLogFormat.Reader; +import org.apache.hudi.common.table.log.block.HoodieAvroDataBlock; +import org.apache.hudi.common.util.FSUtils; import org.springframework.shell.core.CommandMarker; import org.springframework.shell.core.annotation.CliAvailabilityIndicator; import org.springframework.shell.core.annotation.CliCommand; @@ -71,7 +72,7 @@ public class ArchivedCommitsCommand implements CommandMarker { List allStats = new ArrayList<>(); for (FileStatus fs : fsStatuses) { //read the archived file - HoodieLogFormat.Reader reader = HoodieLogFormat.newReader(FSUtils.getFs(basePath, HoodieCLI.conf), + Reader reader = HoodieLogFormat.newReader(FSUtils.getFs(basePath, HoodieCLI.conf), new HoodieLogFile(fs.getPath()), HoodieArchivedMetaEntry.getClassSchema()); List readRecords = new ArrayList<>(); diff --git a/hoodie-cli/src/main/java/com/uber/hoodie/cli/commands/CleansCommand.java b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/CleansCommand.java similarity index 92% rename from hoodie-cli/src/main/java/com/uber/hoodie/cli/commands/CleansCommand.java rename to hudi-cli/src/main/java/org/apache/hudi/cli/commands/CleansCommand.java index c0f7129a8..3217ba55d 100644 --- a/hoodie-cli/src/main/java/com/uber/hoodie/cli/commands/CleansCommand.java +++ b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/CleansCommand.java @@ -16,17 +16,8 @@ * limitations under the License. */ -package com.uber.hoodie.cli.commands; +package org.apache.hudi.cli.commands; -import com.uber.hoodie.avro.model.HoodieCleanMetadata; -import com.uber.hoodie.avro.model.HoodieCleanPartitionMetadata; -import com.uber.hoodie.cli.HoodieCLI; -import com.uber.hoodie.cli.HoodiePrintHelper; -import com.uber.hoodie.cli.TableHeader; -import com.uber.hoodie.common.table.HoodieTimeline; -import com.uber.hoodie.common.table.timeline.HoodieActiveTimeline; -import com.uber.hoodie.common.table.timeline.HoodieInstant; -import com.uber.hoodie.common.util.AvroUtils; import java.io.IOException; import java.util.ArrayList; import java.util.Collections; @@ -34,6 +25,15 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.stream.Collectors; +import org.apache.hudi.avro.model.HoodieCleanMetadata; +import org.apache.hudi.avro.model.HoodieCleanPartitionMetadata; +import org.apache.hudi.cli.HoodieCLI; +import org.apache.hudi.cli.HoodiePrintHelper; +import org.apache.hudi.cli.TableHeader; +import org.apache.hudi.common.table.HoodieTimeline; +import org.apache.hudi.common.table.timeline.HoodieActiveTimeline; +import org.apache.hudi.common.table.timeline.HoodieInstant; +import org.apache.hudi.common.util.AvroUtils; import org.springframework.shell.core.CommandMarker; import org.springframework.shell.core.annotation.CliAvailabilityIndicator; import org.springframework.shell.core.annotation.CliCommand; diff --git a/hoodie-cli/src/main/java/com/uber/hoodie/cli/commands/CommitsCommand.java b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/CommitsCommand.java similarity index 95% rename from hoodie-cli/src/main/java/com/uber/hoodie/cli/commands/CommitsCommand.java rename to hudi-cli/src/main/java/org/apache/hudi/cli/commands/CommitsCommand.java index 2360503bb..8a7420b5a 100644 --- a/hoodie-cli/src/main/java/com/uber/hoodie/cli/commands/CommitsCommand.java +++ b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/CommitsCommand.java @@ -16,20 +16,8 @@ * limitations under the License. */ -package com.uber.hoodie.cli.commands; +package org.apache.hudi.cli.commands; -import com.uber.hoodie.cli.HoodieCLI; -import com.uber.hoodie.cli.HoodiePrintHelper; -import com.uber.hoodie.cli.TableHeader; -import com.uber.hoodie.cli.utils.InputStreamConsumer; -import com.uber.hoodie.cli.utils.SparkUtil; -import com.uber.hoodie.common.model.HoodieCommitMetadata; -import com.uber.hoodie.common.model.HoodieWriteStat; -import com.uber.hoodie.common.table.HoodieTableMetaClient; -import com.uber.hoodie.common.table.HoodieTimeline; -import com.uber.hoodie.common.table.timeline.HoodieActiveTimeline; -import com.uber.hoodie.common.table.timeline.HoodieInstant; -import com.uber.hoodie.common.util.NumericUtils; import java.io.IOException; import java.util.ArrayList; import java.util.Collections; @@ -38,6 +26,18 @@ import java.util.List; import java.util.Map; import java.util.function.Function; import java.util.stream.Collectors; +import org.apache.hudi.cli.HoodieCLI; +import org.apache.hudi.cli.HoodiePrintHelper; +import org.apache.hudi.cli.TableHeader; +import org.apache.hudi.cli.utils.InputStreamConsumer; +import org.apache.hudi.cli.utils.SparkUtil; +import org.apache.hudi.common.model.HoodieCommitMetadata; +import org.apache.hudi.common.model.HoodieWriteStat; +import org.apache.hudi.common.table.HoodieTableMetaClient; +import org.apache.hudi.common.table.HoodieTimeline; +import org.apache.hudi.common.table.timeline.HoodieActiveTimeline; +import org.apache.hudi.common.table.timeline.HoodieInstant; +import org.apache.hudi.common.util.NumericUtils; import org.apache.spark.launcher.SparkLauncher; import org.springframework.shell.core.CommandMarker; import org.springframework.shell.core.annotation.CliAvailabilityIndicator; diff --git a/hoodie-cli/src/main/java/com/uber/hoodie/cli/commands/CompactionCommand.java b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/CompactionCommand.java similarity index 95% rename from hoodie-cli/src/main/java/com/uber/hoodie/cli/commands/CompactionCommand.java rename to hudi-cli/src/main/java/org/apache/hudi/cli/commands/CompactionCommand.java index 89cec5ff2..592d52864 100644 --- a/hoodie-cli/src/main/java/com/uber/hoodie/cli/commands/CompactionCommand.java +++ b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/CompactionCommand.java @@ -16,28 +16,8 @@ * limitations under the License. */ -package com.uber.hoodie.cli.commands; +package org.apache.hudi.cli.commands; -import static com.uber.hoodie.common.table.HoodieTimeline.COMPACTION_ACTION; - -import com.uber.hoodie.CompactionAdminClient.RenameOpResult; -import com.uber.hoodie.CompactionAdminClient.ValidationOpResult; -import com.uber.hoodie.avro.model.HoodieCompactionOperation; -import com.uber.hoodie.avro.model.HoodieCompactionPlan; -import com.uber.hoodie.cli.HoodieCLI; -import com.uber.hoodie.cli.HoodiePrintHelper; -import com.uber.hoodie.cli.TableHeader; -import com.uber.hoodie.cli.commands.SparkMain.SparkCommand; -import com.uber.hoodie.cli.utils.InputStreamConsumer; -import com.uber.hoodie.cli.utils.SparkUtil; -import com.uber.hoodie.common.model.HoodieTableType; -import com.uber.hoodie.common.table.HoodieTimeline; -import com.uber.hoodie.common.table.timeline.HoodieActiveTimeline; -import com.uber.hoodie.common.table.timeline.HoodieInstant; -import com.uber.hoodie.common.table.timeline.HoodieInstant.State; -import com.uber.hoodie.common.util.AvroUtils; -import com.uber.hoodie.common.util.Option; -import com.uber.hoodie.exception.HoodieIOException; import java.io.IOException; import java.io.ObjectInputStream; import java.util.ArrayList; @@ -52,6 +32,24 @@ import java.util.stream.Collectors; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hudi.CompactionAdminClient.RenameOpResult; +import org.apache.hudi.CompactionAdminClient.ValidationOpResult; +import org.apache.hudi.avro.model.HoodieCompactionOperation; +import org.apache.hudi.avro.model.HoodieCompactionPlan; +import org.apache.hudi.cli.HoodieCLI; +import org.apache.hudi.cli.HoodiePrintHelper; +import org.apache.hudi.cli.TableHeader; +import org.apache.hudi.cli.commands.SparkMain.SparkCommand; +import org.apache.hudi.cli.utils.InputStreamConsumer; +import org.apache.hudi.cli.utils.SparkUtil; +import org.apache.hudi.common.model.HoodieTableType; +import org.apache.hudi.common.table.HoodieTimeline; +import org.apache.hudi.common.table.timeline.HoodieActiveTimeline; +import org.apache.hudi.common.table.timeline.HoodieInstant; +import org.apache.hudi.common.table.timeline.HoodieInstant.State; +import org.apache.hudi.common.util.AvroUtils; +import org.apache.hudi.common.util.Option; +import org.apache.hudi.exception.HoodieIOException; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; import org.apache.spark.launcher.SparkLauncher; @@ -98,7 +96,7 @@ public class CompactionCommand implements CommandMarker { for (int i = 0; i < instants.size(); i++) { HoodieInstant instant = instants.get(i); HoodieCompactionPlan workload = null; - if (!instant.getAction().equals(COMPACTION_ACTION)) { + if (!instant.getAction().equals(HoodieTimeline.COMPACTION_ACTION)) { try { // This could be a completed compaction. Assume a compaction request file is present but skip if fails workload = AvroUtils.deserializeCompactionPlan( diff --git a/hoodie-cli/src/main/java/com/uber/hoodie/cli/commands/DatasetsCommand.java b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/DatasetsCommand.java similarity index 91% rename from hoodie-cli/src/main/java/com/uber/hoodie/cli/commands/DatasetsCommand.java rename to hudi-cli/src/main/java/org/apache/hudi/cli/commands/DatasetsCommand.java index 257091ef8..19e1e9525 100644 --- a/hoodie-cli/src/main/java/com/uber/hoodie/cli/commands/DatasetsCommand.java +++ b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/DatasetsCommand.java @@ -16,19 +16,19 @@ * limitations under the License. */ -package com.uber.hoodie.cli.commands; +package org.apache.hudi.cli.commands; -import com.uber.hoodie.cli.HoodieCLI; -import com.uber.hoodie.cli.HoodiePrintHelper; -import com.uber.hoodie.cli.TableHeader; -import com.uber.hoodie.common.model.HoodieTableType; -import com.uber.hoodie.common.table.HoodieTableMetaClient; -import com.uber.hoodie.common.util.ConsistencyGuardConfig; -import com.uber.hoodie.exception.DatasetNotFoundException; import java.io.IOException; import java.util.ArrayList; import java.util.HashMap; import java.util.List; +import org.apache.hudi.cli.HoodieCLI; +import org.apache.hudi.cli.HoodiePrintHelper; +import org.apache.hudi.cli.TableHeader; +import org.apache.hudi.common.model.HoodieTableType; +import org.apache.hudi.common.table.HoodieTableMetaClient; +import org.apache.hudi.common.util.ConsistencyGuardConfig; +import org.apache.hudi.exception.DatasetNotFoundException; import org.springframework.shell.core.CommandMarker; import org.springframework.shell.core.annotation.CliAvailabilityIndicator; import org.springframework.shell.core.annotation.CliCommand; @@ -38,6 +38,10 @@ import org.springframework.stereotype.Component; @Component public class DatasetsCommand implements CommandMarker { + static { + System.out.println("DatasetsCommand getting loaded"); + } + @CliCommand(value = "connect", help = "Connect to a hoodie dataset") public String connect( @CliOption(key = {"path"}, mandatory = true, help = "Base Path of the dataset") final String path, @@ -77,7 +81,7 @@ public class DatasetsCommand implements CommandMarker { @CliOption(key = {"tableName"}, mandatory = true, help = "Hoodie Table Name") final String name, @CliOption(key = {"tableType"}, unspecifiedDefaultValue = "COPY_ON_WRITE", help = "Hoodie Table Type. Must be one of : COPY_ON_WRITE or MERGE_ON_READ") final String tableTypeStr, - @CliOption(key = {"payloadClass"}, unspecifiedDefaultValue = "com.uber.hoodie.common.model.HoodieAvroPayload", + @CliOption(key = {"payloadClass"}, unspecifiedDefaultValue = "org.apache.hudi.common.model.HoodieAvroPayload", help = "Payload Class") final String payloadClass) throws IOException { boolean initialized = HoodieCLI.initConf(); diff --git a/hoodie-cli/src/main/java/com/uber/hoodie/cli/commands/FileSystemViewCommand.java b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/FileSystemViewCommand.java similarity index 95% rename from hoodie-cli/src/main/java/com/uber/hoodie/cli/commands/FileSystemViewCommand.java rename to hudi-cli/src/main/java/org/apache/hudi/cli/commands/FileSystemViewCommand.java index 2cd5d63fa..d86390d7f 100644 --- a/hoodie-cli/src/main/java/com/uber/hoodie/cli/commands/FileSystemViewCommand.java +++ b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/FileSystemViewCommand.java @@ -16,19 +16,8 @@ * limitations under the License. */ -package com.uber.hoodie.cli.commands; +package org.apache.hudi.cli.commands; -import com.uber.hoodie.cli.HoodieCLI; -import com.uber.hoodie.cli.HoodiePrintHelper; -import com.uber.hoodie.cli.TableHeader; -import com.uber.hoodie.common.model.FileSlice; -import com.uber.hoodie.common.table.HoodieTableMetaClient; -import com.uber.hoodie.common.table.HoodieTimeline; -import com.uber.hoodie.common.table.timeline.HoodieDefaultTimeline; -import com.uber.hoodie.common.table.timeline.HoodieInstant; -import com.uber.hoodie.common.table.view.HoodieTableFileSystemView; -import com.uber.hoodie.common.util.NumericUtils; -import com.uber.hoodie.common.util.Option; import java.io.IOException; import java.io.Serializable; import java.util.ArrayList; @@ -42,6 +31,17 @@ import java.util.stream.Stream; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hudi.cli.HoodieCLI; +import org.apache.hudi.cli.HoodiePrintHelper; +import org.apache.hudi.cli.TableHeader; +import org.apache.hudi.common.model.FileSlice; +import org.apache.hudi.common.table.HoodieTableMetaClient; +import org.apache.hudi.common.table.HoodieTimeline; +import org.apache.hudi.common.table.timeline.HoodieDefaultTimeline; +import org.apache.hudi.common.table.timeline.HoodieInstant; +import org.apache.hudi.common.table.view.HoodieTableFileSystemView; +import org.apache.hudi.common.util.NumericUtils; +import org.apache.hudi.common.util.Option; import org.springframework.shell.core.CommandMarker; import org.springframework.shell.core.annotation.CliCommand; import org.springframework.shell.core.annotation.CliOption; diff --git a/hoodie-cli/src/main/java/com/uber/hoodie/cli/commands/HDFSParquetImportCommand.java b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/HDFSParquetImportCommand.java similarity index 91% rename from hoodie-cli/src/main/java/com/uber/hoodie/cli/commands/HDFSParquetImportCommand.java rename to hudi-cli/src/main/java/org/apache/hudi/cli/commands/HDFSParquetImportCommand.java index 24bac0d3f..9ec1b4edd 100644 --- a/hoodie-cli/src/main/java/com/uber/hoodie/cli/commands/HDFSParquetImportCommand.java +++ b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/HDFSParquetImportCommand.java @@ -16,13 +16,13 @@ * limitations under the License. */ -package com.uber.hoodie.cli.commands; +package org.apache.hudi.cli.commands; -import com.uber.hoodie.cli.HoodieCLI; -import com.uber.hoodie.cli.commands.SparkMain.SparkCommand; -import com.uber.hoodie.cli.utils.InputStreamConsumer; -import com.uber.hoodie.cli.utils.SparkUtil; -import com.uber.hoodie.utilities.HDFSParquetImporter; +import org.apache.hudi.cli.HoodieCLI; +import org.apache.hudi.cli.commands.SparkMain.SparkCommand; +import org.apache.hudi.cli.utils.InputStreamConsumer; +import org.apache.hudi.cli.utils.SparkUtil; +import org.apache.hudi.utilities.HDFSParquetImporter.FormatValidator; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; import org.apache.spark.launcher.SparkLauncher; @@ -58,7 +58,7 @@ public class HDFSParquetImportCommand implements CommandMarker { @CliOption(key = "sparkMemory", mandatory = true, help = "Spark executor memory") final String sparkMemory, @CliOption(key = "retry", mandatory = true, help = "Number of retries") final String retry) throws Exception { - (new HDFSParquetImporter.FormatValidator()).validate("format", format); + (new FormatValidator()).validate("format", format); boolean initialized = HoodieCLI.initConf(); HoodieCLI.initFS(initialized); diff --git a/hoodie-cli/src/main/java/com/uber/hoodie/cli/commands/HoodieLogFileCommand.java b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/HoodieLogFileCommand.java similarity index 90% rename from hoodie-cli/src/main/java/com/uber/hoodie/cli/commands/HoodieLogFileCommand.java rename to hudi-cli/src/main/java/org/apache/hudi/cli/commands/HoodieLogFileCommand.java index a7ad9a3cf..9e82cd745 100644 --- a/hoodie-cli/src/main/java/com/uber/hoodie/cli/commands/HoodieLogFileCommand.java +++ b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/HoodieLogFileCommand.java @@ -16,27 +16,10 @@ * limitations under the License. */ -package com.uber.hoodie.cli.commands; +package org.apache.hudi.cli.commands; import com.beust.jcommander.internal.Maps; import com.fasterxml.jackson.databind.ObjectMapper; -import com.uber.hoodie.cli.HoodieCLI; -import com.uber.hoodie.cli.HoodiePrintHelper; -import com.uber.hoodie.cli.TableHeader; -import com.uber.hoodie.common.model.HoodieLogFile; -import com.uber.hoodie.common.model.HoodieRecord; -import com.uber.hoodie.common.model.HoodieRecordPayload; -import com.uber.hoodie.common.table.log.HoodieLogFormat; -import com.uber.hoodie.common.table.log.HoodieMergedLogRecordScanner; -import com.uber.hoodie.common.table.log.block.HoodieAvroDataBlock; -import com.uber.hoodie.common.table.log.block.HoodieCorruptBlock; -import com.uber.hoodie.common.table.log.block.HoodieLogBlock; -import com.uber.hoodie.common.table.log.block.HoodieLogBlock.HeaderMetadataType; -import com.uber.hoodie.common.table.log.block.HoodieLogBlock.HoodieLogBlockType; -import com.uber.hoodie.common.util.Option; -import com.uber.hoodie.config.HoodieCompactionConfig; -import com.uber.hoodie.config.HoodieMemoryConfig; -import com.uber.hoodie.hive.util.SchemaUtil; import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; @@ -49,6 +32,24 @@ import org.apache.avro.generic.IndexedRecord; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hudi.cli.HoodieCLI; +import org.apache.hudi.cli.HoodiePrintHelper; +import org.apache.hudi.cli.TableHeader; +import org.apache.hudi.common.model.HoodieLogFile; +import org.apache.hudi.common.model.HoodieRecord; +import org.apache.hudi.common.model.HoodieRecordPayload; +import org.apache.hudi.common.table.log.HoodieLogFormat; +import org.apache.hudi.common.table.log.HoodieLogFormat.Reader; +import org.apache.hudi.common.table.log.HoodieMergedLogRecordScanner; +import org.apache.hudi.common.table.log.block.HoodieAvroDataBlock; +import org.apache.hudi.common.table.log.block.HoodieCorruptBlock; +import org.apache.hudi.common.table.log.block.HoodieLogBlock; +import org.apache.hudi.common.table.log.block.HoodieLogBlock.HeaderMetadataType; +import org.apache.hudi.common.table.log.block.HoodieLogBlock.HoodieLogBlockType; +import org.apache.hudi.common.util.Option; +import org.apache.hudi.config.HoodieCompactionConfig; +import org.apache.hudi.config.HoodieMemoryConfig; +import org.apache.hudi.hive.util.SchemaUtil; import org.springframework.shell.core.CommandMarker; import org.springframework.shell.core.annotation.CliAvailabilityIndicator; import org.springframework.shell.core.annotation.CliCommand; @@ -90,7 +91,7 @@ public class HoodieLogFileCommand implements CommandMarker { FileStatus[] fsStatus = fs.listStatus(new Path(logFilePath)); Schema writerSchema = new AvroSchemaConverter().convert( SchemaUtil.readSchemaFromLogFile(HoodieCLI.tableMetadata.getFs(), new Path(logFilePath))); - HoodieLogFormat.Reader reader = HoodieLogFormat + Reader reader = HoodieLogFormat .newReader(fs, new HoodieLogFile(fsStatus[0].getPath()), writerSchema); // read the avro blocks diff --git a/hoodie-cli/src/main/java/com/uber/hoodie/cli/commands/HoodieSyncCommand.java b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/HoodieSyncCommand.java similarity index 94% rename from hoodie-cli/src/main/java/com/uber/hoodie/cli/commands/HoodieSyncCommand.java rename to hudi-cli/src/main/java/org/apache/hudi/cli/commands/HoodieSyncCommand.java index 1d9e6d479..614789e2f 100644 --- a/hoodie-cli/src/main/java/com/uber/hoodie/cli/commands/HoodieSyncCommand.java +++ b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/HoodieSyncCommand.java @@ -16,16 +16,16 @@ * limitations under the License. */ -package com.uber.hoodie.cli.commands; +package org.apache.hudi.cli.commands; -import com.uber.hoodie.cli.HoodieCLI; -import com.uber.hoodie.cli.utils.CommitUtil; -import com.uber.hoodie.cli.utils.HiveUtil; -import com.uber.hoodie.common.table.HoodieTableMetaClient; -import com.uber.hoodie.common.table.HoodieTimeline; -import com.uber.hoodie.common.table.timeline.HoodieInstant; import java.util.List; import java.util.stream.Collectors; +import org.apache.hudi.cli.HoodieCLI; +import org.apache.hudi.cli.utils.CommitUtil; +import org.apache.hudi.cli.utils.HiveUtil; +import org.apache.hudi.common.table.HoodieTableMetaClient; +import org.apache.hudi.common.table.HoodieTimeline; +import org.apache.hudi.common.table.timeline.HoodieInstant; import org.springframework.shell.core.CommandMarker; import org.springframework.shell.core.annotation.CliAvailabilityIndicator; import org.springframework.shell.core.annotation.CliCommand; diff --git a/hoodie-cli/src/main/java/com/uber/hoodie/cli/commands/RepairsCommand.java b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/RepairsCommand.java similarity index 92% rename from hoodie-cli/src/main/java/com/uber/hoodie/cli/commands/RepairsCommand.java rename to hudi-cli/src/main/java/org/apache/hudi/cli/commands/RepairsCommand.java index 8967f2b67..ad9b8269a 100644 --- a/hoodie-cli/src/main/java/com/uber/hoodie/cli/commands/RepairsCommand.java +++ b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/RepairsCommand.java @@ -16,17 +16,17 @@ * limitations under the License. */ -package com.uber.hoodie.cli.commands; +package org.apache.hudi.cli.commands; -import com.uber.hoodie.cli.HoodieCLI; -import com.uber.hoodie.cli.HoodiePrintHelper; -import com.uber.hoodie.cli.utils.InputStreamConsumer; -import com.uber.hoodie.cli.utils.SparkUtil; -import com.uber.hoodie.common.model.HoodiePartitionMetadata; -import com.uber.hoodie.common.util.FSUtils; import java.io.IOException; import java.util.List; import org.apache.hadoop.fs.Path; +import org.apache.hudi.cli.HoodieCLI; +import org.apache.hudi.cli.HoodiePrintHelper; +import org.apache.hudi.cli.utils.InputStreamConsumer; +import org.apache.hudi.cli.utils.SparkUtil; +import org.apache.hudi.common.model.HoodiePartitionMetadata; +import org.apache.hudi.common.util.FSUtils; import org.apache.spark.launcher.SparkLauncher; import org.springframework.shell.core.CommandMarker; import org.springframework.shell.core.annotation.CliAvailabilityIndicator; diff --git a/hoodie-cli/src/main/java/com/uber/hoodie/cli/commands/RollbacksCommand.java b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/RollbacksCommand.java similarity index 88% rename from hoodie-cli/src/main/java/com/uber/hoodie/cli/commands/RollbacksCommand.java rename to hudi-cli/src/main/java/org/apache/hudi/cli/commands/RollbacksCommand.java index 9aa1fdaf1..95fc730c3 100644 --- a/hoodie-cli/src/main/java/com/uber/hoodie/cli/commands/RollbacksCommand.java +++ b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/RollbacksCommand.java @@ -16,27 +16,27 @@ * limitations under the License. */ -package com.uber.hoodie.cli.commands; +package org.apache.hudi.cli.commands; -import static com.uber.hoodie.common.table.HoodieTimeline.ROLLBACK_ACTION; +import static org.apache.hudi.common.table.HoodieTimeline.ROLLBACK_ACTION; import com.google.common.collect.ImmutableSet; -import com.uber.hoodie.avro.model.HoodieRollbackMetadata; -import com.uber.hoodie.cli.HoodieCLI; -import com.uber.hoodie.cli.HoodiePrintHelper; -import com.uber.hoodie.cli.TableHeader; -import com.uber.hoodie.common.table.HoodieTableMetaClient; -import com.uber.hoodie.common.table.HoodieTimeline; -import com.uber.hoodie.common.table.timeline.HoodieActiveTimeline; -import com.uber.hoodie.common.table.timeline.HoodieInstant; -import com.uber.hoodie.common.table.timeline.HoodieInstant.State; -import com.uber.hoodie.common.util.AvroUtils; import java.io.IOException; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.stream.Stream; import org.apache.commons.lang3.tuple.Pair; +import org.apache.hudi.avro.model.HoodieRollbackMetadata; +import org.apache.hudi.cli.HoodieCLI; +import org.apache.hudi.cli.HoodiePrintHelper; +import org.apache.hudi.cli.TableHeader; +import org.apache.hudi.common.table.HoodieTableMetaClient; +import org.apache.hudi.common.table.HoodieTimeline; +import org.apache.hudi.common.table.timeline.HoodieActiveTimeline; +import org.apache.hudi.common.table.timeline.HoodieInstant; +import org.apache.hudi.common.table.timeline.HoodieInstant.State; +import org.apache.hudi.common.util.AvroUtils; import org.springframework.shell.core.CommandMarker; import org.springframework.shell.core.annotation.CliCommand; import org.springframework.shell.core.annotation.CliOption; @@ -126,7 +126,7 @@ public class RollbacksCommand implements CommandMarker { class RollbackTimeline extends HoodieActiveTimeline { public RollbackTimeline(HoodieTableMetaClient metaClient) { - super(metaClient, ImmutableSet.builder().add(ROLLBACK_EXTENSION).build()); + super(metaClient, ImmutableSet.builder().add(HoodieTimeline.ROLLBACK_EXTENSION).build()); } } } diff --git a/hoodie-cli/src/main/java/com/uber/hoodie/cli/commands/SavepointsCommand.java b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/SavepointsCommand.java similarity index 91% rename from hoodie-cli/src/main/java/com/uber/hoodie/cli/commands/SavepointsCommand.java rename to hudi-cli/src/main/java/org/apache/hudi/cli/commands/SavepointsCommand.java index 87094690c..8434650b4 100644 --- a/hoodie-cli/src/main/java/com/uber/hoodie/cli/commands/SavepointsCommand.java +++ b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/SavepointsCommand.java @@ -16,23 +16,23 @@ * limitations under the License. */ -package com.uber.hoodie.cli.commands; +package org.apache.hudi.cli.commands; -import com.uber.hoodie.HoodieWriteClient; -import com.uber.hoodie.cli.HoodieCLI; -import com.uber.hoodie.cli.HoodiePrintHelper; -import com.uber.hoodie.cli.utils.InputStreamConsumer; -import com.uber.hoodie.cli.utils.SparkUtil; -import com.uber.hoodie.common.table.HoodieTimeline; -import com.uber.hoodie.common.table.timeline.HoodieActiveTimeline; -import com.uber.hoodie.common.table.timeline.HoodieInstant; -import com.uber.hoodie.config.HoodieIndexConfig; -import com.uber.hoodie.config.HoodieWriteConfig; -import com.uber.hoodie.index.HoodieIndex; import java.io.IOException; import java.util.Collections; import java.util.List; import java.util.stream.Collectors; +import org.apache.hudi.HoodieWriteClient; +import org.apache.hudi.cli.HoodieCLI; +import org.apache.hudi.cli.HoodiePrintHelper; +import org.apache.hudi.cli.utils.InputStreamConsumer; +import org.apache.hudi.cli.utils.SparkUtil; +import org.apache.hudi.common.table.HoodieTimeline; +import org.apache.hudi.common.table.timeline.HoodieActiveTimeline; +import org.apache.hudi.common.table.timeline.HoodieInstant; +import org.apache.hudi.config.HoodieIndexConfig; +import org.apache.hudi.config.HoodieWriteConfig; +import org.apache.hudi.index.HoodieIndex; import org.apache.spark.api.java.JavaSparkContext; import org.apache.spark.launcher.SparkLauncher; import org.springframework.shell.core.CommandMarker; diff --git a/hoodie-cli/src/main/java/com/uber/hoodie/cli/commands/SparkMain.java b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/SparkMain.java similarity index 93% rename from hoodie-cli/src/main/java/com/uber/hoodie/cli/commands/SparkMain.java rename to hudi-cli/src/main/java/org/apache/hudi/cli/commands/SparkMain.java index 9c5d80bfe..2a11fef10 100644 --- a/hoodie-cli/src/main/java/com/uber/hoodie/cli/commands/SparkMain.java +++ b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/SparkMain.java @@ -16,20 +16,21 @@ * limitations under the License. */ -package com.uber.hoodie.cli.commands; +package org.apache.hudi.cli.commands; -import com.uber.hoodie.HoodieWriteClient; -import com.uber.hoodie.cli.DedupeSparkJob; -import com.uber.hoodie.cli.utils.SparkUtil; -import com.uber.hoodie.common.util.FSUtils; -import com.uber.hoodie.config.HoodieIndexConfig; -import com.uber.hoodie.config.HoodieWriteConfig; -import com.uber.hoodie.index.HoodieIndex; -import com.uber.hoodie.io.compact.strategy.UnBoundedCompactionStrategy; -import com.uber.hoodie.utilities.HDFSParquetImporter; -import com.uber.hoodie.utilities.HoodieCompactionAdminTool; -import com.uber.hoodie.utilities.HoodieCompactionAdminTool.Operation; -import com.uber.hoodie.utilities.HoodieCompactor; +import org.apache.hudi.HoodieWriteClient; +import org.apache.hudi.cli.DedupeSparkJob; +import org.apache.hudi.cli.utils.SparkUtil; +import org.apache.hudi.common.util.FSUtils; +import org.apache.hudi.config.HoodieIndexConfig; +import org.apache.hudi.config.HoodieWriteConfig; +import org.apache.hudi.index.HoodieIndex; +import org.apache.hudi.io.compact.strategy.UnBoundedCompactionStrategy; +import org.apache.hudi.utilities.HDFSParquetImporter; +import org.apache.hudi.utilities.HDFSParquetImporter.Config; +import org.apache.hudi.utilities.HoodieCompactionAdminTool; +import org.apache.hudi.utilities.HoodieCompactionAdminTool.Operation; +import org.apache.hudi.utilities.HoodieCompactor; import org.apache.log4j.Logger; import org.apache.spark.api.java.JavaSparkContext; import org.apache.spark.sql.SQLContext; @@ -116,7 +117,7 @@ public class SparkMain { String srcPath, String targetPath, String tableName, String tableType, String rowKey, String partitionKey, int parallelism, String schemaFile, String sparkMaster, String sparkMemory, int retry) throws Exception { - HDFSParquetImporter.Config cfg = new HDFSParquetImporter.Config(); + Config cfg = new Config(); cfg.command = command; cfg.srcPath = srcPath; cfg.targetPath = targetPath; diff --git a/hoodie-cli/src/main/java/com/uber/hoodie/cli/commands/StatsCommand.java b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/StatsCommand.java similarity index 93% rename from hoodie-cli/src/main/java/com/uber/hoodie/cli/commands/StatsCommand.java rename to hudi-cli/src/main/java/org/apache/hudi/cli/commands/StatsCommand.java index 36f768291..9b8f41cb9 100644 --- a/hoodie-cli/src/main/java/com/uber/hoodie/cli/commands/StatsCommand.java +++ b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/StatsCommand.java @@ -16,20 +16,11 @@ * limitations under the License. */ -package com.uber.hoodie.cli.commands; +package org.apache.hudi.cli.commands; import com.codahale.metrics.Histogram; import com.codahale.metrics.Snapshot; import com.codahale.metrics.UniformReservoir; -import com.uber.hoodie.cli.HoodieCLI; -import com.uber.hoodie.cli.HoodiePrintHelper; -import com.uber.hoodie.cli.TableHeader; -import com.uber.hoodie.common.model.HoodieCommitMetadata; -import com.uber.hoodie.common.table.HoodieTimeline; -import com.uber.hoodie.common.table.timeline.HoodieActiveTimeline; -import com.uber.hoodie.common.table.timeline.HoodieInstant; -import com.uber.hoodie.common.util.FSUtils; -import com.uber.hoodie.common.util.NumericUtils; import java.io.IOException; import java.text.DecimalFormat; import java.util.ArrayList; @@ -41,6 +32,15 @@ import java.util.stream.Collectors; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hudi.cli.HoodieCLI; +import org.apache.hudi.cli.HoodiePrintHelper; +import org.apache.hudi.cli.TableHeader; +import org.apache.hudi.common.model.HoodieCommitMetadata; +import org.apache.hudi.common.table.HoodieTimeline; +import org.apache.hudi.common.table.timeline.HoodieActiveTimeline; +import org.apache.hudi.common.table.timeline.HoodieInstant; +import org.apache.hudi.common.util.FSUtils; +import org.apache.hudi.common.util.NumericUtils; import org.springframework.shell.core.CommandMarker; import org.springframework.shell.core.annotation.CliAvailabilityIndicator; import org.springframework.shell.core.annotation.CliCommand; diff --git a/hoodie-cli/src/main/java/com/uber/hoodie/cli/commands/UtilsCommand.java b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/UtilsCommand.java similarity index 97% rename from hoodie-cli/src/main/java/com/uber/hoodie/cli/commands/UtilsCommand.java rename to hudi-cli/src/main/java/org/apache/hudi/cli/commands/UtilsCommand.java index c9be3a6cd..514a3f88a 100644 --- a/hoodie-cli/src/main/java/com/uber/hoodie/cli/commands/UtilsCommand.java +++ b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/UtilsCommand.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package com.uber.hoodie.cli.commands; +package org.apache.hudi.cli.commands; import org.springframework.shell.core.CommandMarker; import org.springframework.shell.core.annotation.CliCommand; diff --git a/hoodie-cli/src/main/java/com/uber/hoodie/cli/utils/CommitUtil.java b/hudi-cli/src/main/java/org/apache/hudi/cli/utils/CommitUtil.java similarity index 85% rename from hoodie-cli/src/main/java/com/uber/hoodie/cli/utils/CommitUtil.java rename to hudi-cli/src/main/java/org/apache/hudi/cli/utils/CommitUtil.java index 4e36fbe0e..26568e7e5 100644 --- a/hoodie-cli/src/main/java/com/uber/hoodie/cli/utils/CommitUtil.java +++ b/hudi-cli/src/main/java/org/apache/hudi/cli/utils/CommitUtil.java @@ -16,14 +16,14 @@ * limitations under the License. */ -package com.uber.hoodie.cli.utils; +package org.apache.hudi.cli.utils; -import com.uber.hoodie.common.model.HoodieCommitMetadata; -import com.uber.hoodie.common.table.HoodieTableMetaClient; -import com.uber.hoodie.common.table.HoodieTimeline; -import com.uber.hoodie.common.table.timeline.HoodieInstant; import java.io.IOException; import java.util.List; +import org.apache.hudi.common.model.HoodieCommitMetadata; +import org.apache.hudi.common.table.HoodieTableMetaClient; +import org.apache.hudi.common.table.HoodieTimeline; +import org.apache.hudi.common.table.timeline.HoodieInstant; public class CommitUtil { diff --git a/hoodie-cli/src/main/java/com/uber/hoodie/cli/utils/HiveUtil.java b/hudi-cli/src/main/java/org/apache/hudi/cli/utils/HiveUtil.java similarity index 98% rename from hoodie-cli/src/main/java/com/uber/hoodie/cli/utils/HiveUtil.java rename to hudi-cli/src/main/java/org/apache/hudi/cli/utils/HiveUtil.java index 8e11c2bd6..97fd85e9d 100644 --- a/hoodie-cli/src/main/java/com/uber/hoodie/cli/utils/HiveUtil.java +++ b/hudi-cli/src/main/java/org/apache/hudi/cli/utils/HiveUtil.java @@ -16,15 +16,15 @@ * limitations under the License. */ -package com.uber.hoodie.cli.utils; +package org.apache.hudi.cli.utils; -import com.uber.hoodie.common.table.HoodieTableMetaClient; import java.sql.Connection; import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; import javax.sql.DataSource; import org.apache.commons.dbcp.BasicDataSource; +import org.apache.hudi.common.table.HoodieTableMetaClient; import org.joda.time.DateTime; public class HiveUtil { diff --git a/hoodie-cli/src/main/java/com/uber/hoodie/cli/utils/InputStreamConsumer.java b/hudi-cli/src/main/java/org/apache/hudi/cli/utils/InputStreamConsumer.java similarity index 98% rename from hoodie-cli/src/main/java/com/uber/hoodie/cli/utils/InputStreamConsumer.java rename to hudi-cli/src/main/java/org/apache/hudi/cli/utils/InputStreamConsumer.java index cc2291132..9d53c567f 100644 --- a/hoodie-cli/src/main/java/com/uber/hoodie/cli/utils/InputStreamConsumer.java +++ b/hudi-cli/src/main/java/org/apache/hudi/cli/utils/InputStreamConsumer.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package com.uber.hoodie.cli.utils; +package org.apache.hudi.cli.utils; import java.io.BufferedReader; import java.io.IOException; diff --git a/hoodie-cli/src/main/java/com/uber/hoodie/cli/utils/SparkUtil.java b/hudi-cli/src/main/java/org/apache/hudi/cli/utils/SparkUtil.java similarity index 95% rename from hoodie-cli/src/main/java/com/uber/hoodie/cli/utils/SparkUtil.java rename to hudi-cli/src/main/java/org/apache/hudi/cli/utils/SparkUtil.java index e391e97f6..69ac79620 100644 --- a/hoodie-cli/src/main/java/com/uber/hoodie/cli/utils/SparkUtil.java +++ b/hudi-cli/src/main/java/org/apache/hudi/cli/utils/SparkUtil.java @@ -16,15 +16,14 @@ * limitations under the License. */ -package com.uber.hoodie.cli.utils; +package org.apache.hudi.cli.utils; -import com.uber.hoodie.HoodieWriteClient; -import com.uber.hoodie.cli.commands.SparkMain; -import com.uber.hoodie.common.util.FSUtils; import java.io.File; import java.net.URISyntaxException; - import org.apache.commons.lang.StringUtils; +import org.apache.hudi.HoodieWriteClient; +import org.apache.hudi.cli.commands.SparkMain; +import org.apache.hudi.common.util.FSUtils; import org.apache.log4j.Logger; import org.apache.spark.SparkConf; import org.apache.spark.api.java.JavaSparkContext; diff --git a/hoodie-cli/src/main/resources/META-INF/LICENSE.txt b/hudi-cli/src/main/resources/META-INF/LICENSE.txt similarity index 100% rename from hoodie-cli/src/main/resources/META-INF/LICENSE.txt rename to hudi-cli/src/main/resources/META-INF/LICENSE.txt diff --git a/hoodie-cli/src/main/resources/META-INF/NOTICE.txt b/hudi-cli/src/main/resources/META-INF/NOTICE.txt similarity index 100% rename from hoodie-cli/src/main/resources/META-INF/NOTICE.txt rename to hudi-cli/src/main/resources/META-INF/NOTICE.txt diff --git a/hoodie-cli/src/main/resources/META-INF/spring/spring-shell-plugin.xml b/hudi-cli/src/main/resources/META-INF/spring/spring-shell-plugin.xml similarity index 95% rename from hoodie-cli/src/main/resources/META-INF/spring/spring-shell-plugin.xml rename to hudi-cli/src/main/resources/META-INF/spring/spring-shell-plugin.xml index cfba17f7f..e28b4f9e8 100644 --- a/hoodie-cli/src/main/resources/META-INF/spring/spring-shell-plugin.xml +++ b/hudi-cli/src/main/resources/META-INF/spring/spring-shell-plugin.xml @@ -23,6 +23,6 @@ xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd http://www.springframework.org/schema/context http://www.springframework.org/schema/context/spring-context.xsd"> - + diff --git a/hoodie-cli/src/main/scala/com/uber/hoodie/cli/DedupeSparkJob.scala b/hudi-cli/src/main/scala/org/apache/hudi/cli/DedupeSparkJob.scala similarity index 95% rename from hoodie-cli/src/main/scala/com/uber/hoodie/cli/DedupeSparkJob.scala rename to hudi-cli/src/main/scala/org/apache/hudi/cli/DedupeSparkJob.scala index 9714abc0e..c9966c26f 100644 --- a/hoodie-cli/src/main/scala/com/uber/hoodie/cli/DedupeSparkJob.scala +++ b/hudi-cli/src/main/scala/org/apache/hudi/cli/DedupeSparkJob.scala @@ -15,16 +15,16 @@ * limitations under the License. */ -package com.uber.hoodie.cli +package org.apache.hudi.cli import java.util.stream.Collectors -import com.uber.hoodie.common.model.{HoodieDataFile, HoodieRecord} -import com.uber.hoodie.common.table.HoodieTableMetaClient -import com.uber.hoodie.common.table.view.HoodieTableFileSystemView -import com.uber.hoodie.common.util.FSUtils -import com.uber.hoodie.exception.HoodieException import org.apache.hadoop.fs.{FileSystem, FileUtil, Path} +import org.apache.hudi.common.model.{HoodieDataFile, HoodieRecord} +import org.apache.hudi.common.table.HoodieTableMetaClient +import org.apache.hudi.common.table.view.HoodieTableFileSystemView +import org.apache.hudi.common.util.FSUtils +import org.apache.hudi.exception.HoodieException import org.apache.log4j.Logger import org.apache.spark.sql.{DataFrame, SQLContext} diff --git a/hoodie-cli/src/main/scala/com/uber/hoodie/cli/SparkHelpers.scala b/hudi-cli/src/main/scala/org/apache/hudi/cli/SparkHelpers.scala similarity index 91% rename from hoodie-cli/src/main/scala/com/uber/hoodie/cli/SparkHelpers.scala rename to hudi-cli/src/main/scala/org/apache/hudi/cli/SparkHelpers.scala index 57d4f3388..5f78038d8 100644 --- a/hoodie-cli/src/main/scala/com/uber/hoodie/cli/SparkHelpers.scala +++ b/hudi-cli/src/main/scala/org/apache/hudi/cli/SparkHelpers.scala @@ -15,18 +15,18 @@ * limitations under the License. */ -package com.uber.hoodie.cli +package org.apache.hudi.cli -import com.uber.hoodie.avro.HoodieAvroWriteSupport -import com.uber.hoodie.common.model.HoodieRecord -import com.uber.hoodie.common.util.ParquetUtils -import com.uber.hoodie.common.{BloomFilter, HoodieJsonPayload} -import com.uber.hoodie.config.{HoodieIndexConfig, HoodieStorageConfig} -import com.uber.hoodie.io.storage.{HoodieParquetConfig, HoodieParquetWriter} import org.apache.avro.Schema import org.apache.avro.generic.IndexedRecord import org.apache.hadoop.conf.Configuration import org.apache.hadoop.fs.{FileSystem, Path} +import org.apache.hudi.avro.HoodieAvroWriteSupport +import org.apache.hudi.common.model.HoodieRecord +import org.apache.hudi.common.util.ParquetUtils +import org.apache.hudi.common.{BloomFilter, HoodieJsonPayload} +import org.apache.hudi.config.{HoodieIndexConfig, HoodieStorageConfig} +import org.apache.hudi.io.storage.{HoodieParquetConfig, HoodieParquetWriter} import org.apache.parquet.avro.AvroSchemaConverter import org.apache.parquet.hadoop.ParquetFileReader import org.apache.parquet.hadoop.metadata.CompressionCodecName @@ -123,7 +123,7 @@ class SparkHelper(sqlContext: SQLContext, fs: FileSystem) { */ def fileKeysAgainstBF(conf: Configuration, sqlContext: SQLContext, file: String): Boolean = { val bfStr = SparkHelpers.getBloomFilter(file, conf) - val bf = new com.uber.hoodie.common.BloomFilter(bfStr) + val bf = new BloomFilter(bfStr) val foundCount = sqlContext.parquetFile(file) .select(s"`${HoodieRecord.RECORD_KEY_METADATA_FIELD}`") .collect().count(r => !bf.mightContain(r.getString(0))) diff --git a/hoodie-client/pom.xml b/hudi-client/pom.xml similarity index 92% rename from hoodie-client/pom.xml rename to hudi-client/pom.xml index df08b1b86..95959ef5c 100644 --- a/hoodie-client/pom.xml +++ b/hudi-client/pom.xml @@ -17,13 +17,13 @@ --> - hoodie - com.uber.hoodie - 0.4.8-SNAPSHOT + hudi + org.apache.hudi + 0.5.0-SNAPSHOT 4.0.0 - hoodie-client + hudi-client jar @@ -69,13 +69,13 @@ - com.uber.hoodie - hoodie-common + org.apache.hudi + hudi-common ${project.version} - com.uber.hoodie - hoodie-timeline-service + org.apache.hudi + hudi-timeline-service ${project.version} @@ -118,8 +118,8 @@ - com.uber.hoodie - hoodie-common + org.apache.hudi + hudi-common ${project.version} tests test-jar @@ -195,8 +195,8 @@ test - com.uber.hoodie - hoodie-hadoop-mr + org.apache.hudi + hudi-hadoop-mr ${project.version} test diff --git a/hoodie-client/src/main/java/com/uber/hoodie/AbstractHoodieClient.java b/hudi-client/src/main/java/org/apache/hudi/AbstractHoodieClient.java similarity index 93% rename from hoodie-client/src/main/java/com/uber/hoodie/AbstractHoodieClient.java rename to hudi-client/src/main/java/org/apache/hudi/AbstractHoodieClient.java index aae8e8030..ff4cf6f06 100644 --- a/hoodie-client/src/main/java/com/uber/hoodie/AbstractHoodieClient.java +++ b/hudi-client/src/main/java/org/apache/hudi/AbstractHoodieClient.java @@ -16,17 +16,17 @@ * limitations under the License. */ -package com.uber.hoodie; +package org.apache.hudi; -import com.uber.hoodie.client.embedded.EmbeddedTimelineService; -import com.uber.hoodie.client.utils.ClientUtils; -import com.uber.hoodie.common.table.HoodieTableMetaClient; -import com.uber.hoodie.common.util.FSUtils; -import com.uber.hoodie.common.util.Option; -import com.uber.hoodie.config.HoodieWriteConfig; import java.io.IOException; import java.io.Serializable; import org.apache.hadoop.fs.FileSystem; +import org.apache.hudi.client.embedded.EmbeddedTimelineService; +import org.apache.hudi.client.utils.ClientUtils; +import org.apache.hudi.common.table.HoodieTableMetaClient; +import org.apache.hudi.common.util.FSUtils; +import org.apache.hudi.common.util.Option; +import org.apache.hudi.config.HoodieWriteConfig; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; import org.apache.spark.api.java.JavaSparkContext; diff --git a/hoodie-client/src/main/java/com/uber/hoodie/CompactionAdminClient.java b/hudi-client/src/main/java/org/apache/hudi/CompactionAdminClient.java similarity index 95% rename from hoodie-client/src/main/java/com/uber/hoodie/CompactionAdminClient.java rename to hudi-client/src/main/java/org/apache/hudi/CompactionAdminClient.java index dc8da0d9c..d3ac06d5f 100644 --- a/hoodie-client/src/main/java/com/uber/hoodie/CompactionAdminClient.java +++ b/hudi-client/src/main/java/org/apache/hudi/CompactionAdminClient.java @@ -16,34 +16,11 @@ * limitations under the License. */ -package com.uber.hoodie; +package org.apache.hudi; -import static com.uber.hoodie.common.table.HoodieTimeline.COMPACTION_ACTION; +import static org.apache.hudi.common.table.HoodieTimeline.COMPACTION_ACTION; import com.google.common.base.Preconditions; -import com.uber.hoodie.avro.model.HoodieCompactionOperation; -import com.uber.hoodie.avro.model.HoodieCompactionPlan; -import com.uber.hoodie.client.embedded.EmbeddedTimelineService; -import com.uber.hoodie.common.model.CompactionOperation; -import com.uber.hoodie.common.model.FileSlice; -import com.uber.hoodie.common.model.HoodieDataFile; -import com.uber.hoodie.common.model.HoodieFileGroupId; -import com.uber.hoodie.common.model.HoodieLogFile; -import com.uber.hoodie.common.table.HoodieTableMetaClient; -import com.uber.hoodie.common.table.HoodieTimeline; -import com.uber.hoodie.common.table.log.HoodieLogFormat; -import com.uber.hoodie.common.table.timeline.HoodieInstant; -import com.uber.hoodie.common.table.timeline.HoodieInstant.State; -import com.uber.hoodie.common.table.view.HoodieTableFileSystemView; -import com.uber.hoodie.common.util.AvroUtils; -import com.uber.hoodie.common.util.CompactionUtils; -import com.uber.hoodie.common.util.FSUtils; -import com.uber.hoodie.common.util.Option; -import com.uber.hoodie.common.util.collection.Pair; -import com.uber.hoodie.config.HoodieWriteConfig; -import com.uber.hoodie.exception.HoodieException; -import com.uber.hoodie.exception.HoodieIOException; -import com.uber.hoodie.func.OperationResult; import java.io.FileNotFoundException; import java.io.IOException; import java.io.Serializable; @@ -54,6 +31,29 @@ import java.util.Set; import java.util.stream.Collectors; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.Path; +import org.apache.hudi.avro.model.HoodieCompactionOperation; +import org.apache.hudi.avro.model.HoodieCompactionPlan; +import org.apache.hudi.client.embedded.EmbeddedTimelineService; +import org.apache.hudi.common.model.CompactionOperation; +import org.apache.hudi.common.model.FileSlice; +import org.apache.hudi.common.model.HoodieDataFile; +import org.apache.hudi.common.model.HoodieFileGroupId; +import org.apache.hudi.common.model.HoodieLogFile; +import org.apache.hudi.common.table.HoodieTableMetaClient; +import org.apache.hudi.common.table.HoodieTimeline; +import org.apache.hudi.common.table.log.HoodieLogFormat; +import org.apache.hudi.common.table.timeline.HoodieInstant; +import org.apache.hudi.common.table.timeline.HoodieInstant.State; +import org.apache.hudi.common.table.view.HoodieTableFileSystemView; +import org.apache.hudi.common.util.AvroUtils; +import org.apache.hudi.common.util.CompactionUtils; +import org.apache.hudi.common.util.FSUtils; +import org.apache.hudi.common.util.Option; +import org.apache.hudi.common.util.collection.Pair; +import org.apache.hudi.config.HoodieWriteConfig; +import org.apache.hudi.exception.HoodieException; +import org.apache.hudi.exception.HoodieIOException; +import org.apache.hudi.func.OperationResult; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; import org.apache.spark.api.java.JavaSparkContext; diff --git a/hoodie-client/src/main/java/com/uber/hoodie/HoodieReadClient.java b/hudi-client/src/main/java/org/apache/hudi/HoodieReadClient.java similarity index 90% rename from hoodie-client/src/main/java/com/uber/hoodie/HoodieReadClient.java rename to hudi-client/src/main/java/org/apache/hudi/HoodieReadClient.java index f660864aa..43f4c5760 100644 --- a/hoodie-client/src/main/java/com/uber/hoodie/HoodieReadClient.java +++ b/hudi-client/src/main/java/org/apache/hudi/HoodieReadClient.java @@ -16,28 +16,28 @@ * limitations under the License. */ -package com.uber.hoodie; +package org.apache.hudi; -import com.uber.hoodie.avro.model.HoodieCompactionPlan; -import com.uber.hoodie.client.embedded.EmbeddedTimelineService; -import com.uber.hoodie.common.model.HoodieDataFile; -import com.uber.hoodie.common.model.HoodieKey; -import com.uber.hoodie.common.model.HoodieRecord; -import com.uber.hoodie.common.model.HoodieRecordPayload; -import com.uber.hoodie.common.table.HoodieTableMetaClient; -import com.uber.hoodie.common.table.HoodieTimeline; -import com.uber.hoodie.common.util.CompactionUtils; -import com.uber.hoodie.common.util.Option; -import com.uber.hoodie.common.util.collection.Pair; -import com.uber.hoodie.config.HoodieIndexConfig; -import com.uber.hoodie.config.HoodieWriteConfig; -import com.uber.hoodie.exception.HoodieIndexException; -import com.uber.hoodie.index.HoodieIndex; -import com.uber.hoodie.table.HoodieTable; import java.util.HashSet; import java.util.List; import java.util.Set; import java.util.stream.Collectors; +import org.apache.hudi.avro.model.HoodieCompactionPlan; +import org.apache.hudi.client.embedded.EmbeddedTimelineService; +import org.apache.hudi.common.model.HoodieDataFile; +import org.apache.hudi.common.model.HoodieKey; +import org.apache.hudi.common.model.HoodieRecord; +import org.apache.hudi.common.model.HoodieRecordPayload; +import org.apache.hudi.common.table.HoodieTableMetaClient; +import org.apache.hudi.common.table.HoodieTimeline; +import org.apache.hudi.common.util.CompactionUtils; +import org.apache.hudi.common.util.Option; +import org.apache.hudi.common.util.collection.Pair; +import org.apache.hudi.config.HoodieIndexConfig; +import org.apache.hudi.config.HoodieWriteConfig; +import org.apache.hudi.exception.HoodieIndexException; +import org.apache.hudi.index.HoodieIndex; +import org.apache.hudi.table.HoodieTable; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; import org.apache.spark.SparkConf; diff --git a/hoodie-client/src/main/java/com/uber/hoodie/HoodieWriteClient.java b/hudi-client/src/main/java/org/apache/hudi/HoodieWriteClient.java similarity index 96% rename from hoodie-client/src/main/java/com/uber/hoodie/HoodieWriteClient.java rename to hudi-client/src/main/java/org/apache/hudi/HoodieWriteClient.java index 10fc61249..d81ecaff5 100644 --- a/hoodie-client/src/main/java/com/uber/hoodie/HoodieWriteClient.java +++ b/hudi-client/src/main/java/org/apache/hudi/HoodieWriteClient.java @@ -16,55 +16,12 @@ * limitations under the License. */ -package com.uber.hoodie; +package org.apache.hudi; import com.codahale.metrics.Timer; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableMap; -import com.uber.hoodie.avro.model.HoodieCleanMetadata; -import com.uber.hoodie.avro.model.HoodieCompactionPlan; -import com.uber.hoodie.avro.model.HoodieRestoreMetadata; -import com.uber.hoodie.avro.model.HoodieRollbackMetadata; -import com.uber.hoodie.avro.model.HoodieSavepointMetadata; -import com.uber.hoodie.client.embedded.EmbeddedTimelineService; -import com.uber.hoodie.common.HoodieCleanStat; -import com.uber.hoodie.common.HoodieRollbackStat; -import com.uber.hoodie.common.model.HoodieCommitMetadata; -import com.uber.hoodie.common.model.HoodieDataFile; -import com.uber.hoodie.common.model.HoodieKey; -import com.uber.hoodie.common.model.HoodieRecord; -import com.uber.hoodie.common.model.HoodieRecordPayload; -import com.uber.hoodie.common.model.HoodieRollingStat; -import com.uber.hoodie.common.model.HoodieRollingStatMetadata; -import com.uber.hoodie.common.model.HoodieTableType; -import com.uber.hoodie.common.model.HoodieWriteStat; -import com.uber.hoodie.common.table.HoodieTableMetaClient; -import com.uber.hoodie.common.table.HoodieTimeline; -import com.uber.hoodie.common.table.TableFileSystemView; -import com.uber.hoodie.common.table.timeline.HoodieActiveTimeline; -import com.uber.hoodie.common.table.timeline.HoodieInstant; -import com.uber.hoodie.common.table.timeline.HoodieInstant.State; -import com.uber.hoodie.common.util.AvroUtils; -import com.uber.hoodie.common.util.FSUtils; -import com.uber.hoodie.common.util.Option; -import com.uber.hoodie.config.HoodieCompactionConfig; -import com.uber.hoodie.config.HoodieWriteConfig; -import com.uber.hoodie.exception.HoodieCommitException; -import com.uber.hoodie.exception.HoodieCompactionException; -import com.uber.hoodie.exception.HoodieIOException; -import com.uber.hoodie.exception.HoodieInsertException; -import com.uber.hoodie.exception.HoodieRollbackException; -import com.uber.hoodie.exception.HoodieSavepointException; -import com.uber.hoodie.exception.HoodieUpsertException; -import com.uber.hoodie.func.BulkInsertMapFunction; -import com.uber.hoodie.index.HoodieIndex; -import com.uber.hoodie.io.HoodieCommitArchiveLog; -import com.uber.hoodie.metrics.HoodieMetrics; -import com.uber.hoodie.table.HoodieTable; -import com.uber.hoodie.table.UserDefinedBulkInsertPartitioner; -import com.uber.hoodie.table.WorkloadProfile; -import com.uber.hoodie.table.WorkloadStat; import java.io.IOException; import java.nio.charset.StandardCharsets; import java.text.ParseException; @@ -76,6 +33,49 @@ import java.util.List; import java.util.Map; import java.util.stream.Collectors; import java.util.stream.IntStream; +import org.apache.hudi.avro.model.HoodieCleanMetadata; +import org.apache.hudi.avro.model.HoodieCompactionPlan; +import org.apache.hudi.avro.model.HoodieRestoreMetadata; +import org.apache.hudi.avro.model.HoodieRollbackMetadata; +import org.apache.hudi.avro.model.HoodieSavepointMetadata; +import org.apache.hudi.client.embedded.EmbeddedTimelineService; +import org.apache.hudi.common.HoodieCleanStat; +import org.apache.hudi.common.HoodieRollbackStat; +import org.apache.hudi.common.model.HoodieCommitMetadata; +import org.apache.hudi.common.model.HoodieDataFile; +import org.apache.hudi.common.model.HoodieKey; +import org.apache.hudi.common.model.HoodieRecord; +import org.apache.hudi.common.model.HoodieRecordPayload; +import org.apache.hudi.common.model.HoodieRollingStat; +import org.apache.hudi.common.model.HoodieRollingStatMetadata; +import org.apache.hudi.common.model.HoodieTableType; +import org.apache.hudi.common.model.HoodieWriteStat; +import org.apache.hudi.common.table.HoodieTableMetaClient; +import org.apache.hudi.common.table.HoodieTimeline; +import org.apache.hudi.common.table.TableFileSystemView.ReadOptimizedView; +import org.apache.hudi.common.table.timeline.HoodieActiveTimeline; +import org.apache.hudi.common.table.timeline.HoodieInstant; +import org.apache.hudi.common.table.timeline.HoodieInstant.State; +import org.apache.hudi.common.util.AvroUtils; +import org.apache.hudi.common.util.FSUtils; +import org.apache.hudi.common.util.Option; +import org.apache.hudi.config.HoodieCompactionConfig; +import org.apache.hudi.config.HoodieWriteConfig; +import org.apache.hudi.exception.HoodieCommitException; +import org.apache.hudi.exception.HoodieCompactionException; +import org.apache.hudi.exception.HoodieIOException; +import org.apache.hudi.exception.HoodieInsertException; +import org.apache.hudi.exception.HoodieRollbackException; +import org.apache.hudi.exception.HoodieSavepointException; +import org.apache.hudi.exception.HoodieUpsertException; +import org.apache.hudi.func.BulkInsertMapFunction; +import org.apache.hudi.index.HoodieIndex; +import org.apache.hudi.io.HoodieCommitArchiveLog; +import org.apache.hudi.metrics.HoodieMetrics; +import org.apache.hudi.table.HoodieTable; +import org.apache.hudi.table.UserDefinedBulkInsertPartitioner; +import org.apache.hudi.table.WorkloadProfile; +import org.apache.hudi.table.WorkloadStat; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; import org.apache.spark.Partitioner; @@ -644,7 +644,7 @@ public class HoodieWriteClient extends AbstractHo .mapToPair((PairFunction>) partitionPath -> { // Scan all partitions files with this commit time logger.info("Collecting latest files in partition path " + partitionPath); - TableFileSystemView.ReadOptimizedView view = table.getROFileSystemView(); + ReadOptimizedView view = table.getROFileSystemView(); List latestFiles = view.getLatestDataFilesBeforeOrOn(partitionPath, commitTime) .map(HoodieDataFile::getFileName).collect(Collectors.toList()); return new Tuple2<>(partitionPath, latestFiles); diff --git a/hoodie-client/src/main/java/com/uber/hoodie/WriteStatus.java b/hudi-client/src/main/java/org/apache/hudi/WriteStatus.java similarity index 96% rename from hoodie-client/src/main/java/com/uber/hoodie/WriteStatus.java rename to hudi-client/src/main/java/org/apache/hudi/WriteStatus.java index 55a476c78..ad488bb4a 100644 --- a/hoodie-client/src/main/java/com/uber/hoodie/WriteStatus.java +++ b/hudi-client/src/main/java/org/apache/hudi/WriteStatus.java @@ -16,18 +16,18 @@ * limitations under the License. */ -package com.uber.hoodie; +package org.apache.hudi; -import com.uber.hoodie.common.model.HoodieKey; -import com.uber.hoodie.common.model.HoodieRecord; -import com.uber.hoodie.common.model.HoodieWriteStat; -import com.uber.hoodie.common.util.Option; import java.io.Serializable; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Random; +import org.apache.hudi.common.model.HoodieKey; +import org.apache.hudi.common.model.HoodieRecord; +import org.apache.hudi.common.model.HoodieWriteStat; +import org.apache.hudi.common.util.Option; /** * Status of a write operation. diff --git a/hoodie-client/src/main/java/com/uber/hoodie/client/embedded/EmbeddedTimelineService.java b/hudi-client/src/main/java/org/apache/hudi/client/embedded/EmbeddedTimelineService.java similarity index 90% rename from hoodie-client/src/main/java/com/uber/hoodie/client/embedded/EmbeddedTimelineService.java rename to hudi-client/src/main/java/org/apache/hudi/client/embedded/EmbeddedTimelineService.java index 4c0539a54..4c6089c8a 100644 --- a/hoodie-client/src/main/java/com/uber/hoodie/client/embedded/EmbeddedTimelineService.java +++ b/hudi-client/src/main/java/org/apache/hudi/client/embedded/EmbeddedTimelineService.java @@ -16,16 +16,16 @@ * limitations under the License. */ -package com.uber.hoodie.client.embedded; +package org.apache.hudi.client.embedded; -import com.uber.hoodie.common.SerializableConfiguration; -import com.uber.hoodie.common.table.view.FileSystemViewManager; -import com.uber.hoodie.common.table.view.FileSystemViewStorageConfig; -import com.uber.hoodie.common.table.view.FileSystemViewStorageType; -import com.uber.hoodie.common.util.NetworkUtils; -import com.uber.hoodie.timeline.service.TimelineService; import java.io.IOException; import org.apache.hadoop.conf.Configuration; +import org.apache.hudi.common.SerializableConfiguration; +import org.apache.hudi.common.table.view.FileSystemViewManager; +import org.apache.hudi.common.table.view.FileSystemViewStorageConfig; +import org.apache.hudi.common.table.view.FileSystemViewStorageType; +import org.apache.hudi.common.util.NetworkUtils; +import org.apache.hudi.timeline.service.TimelineService; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; import org.apache.spark.SparkConf; diff --git a/hoodie-client/src/main/java/com/uber/hoodie/client/utils/ClientUtils.java b/hudi-client/src/main/java/org/apache/hudi/client/utils/ClientUtils.java similarity index 90% rename from hoodie-client/src/main/java/com/uber/hoodie/client/utils/ClientUtils.java rename to hudi-client/src/main/java/org/apache/hudi/client/utils/ClientUtils.java index e9859f914..2c45b1362 100644 --- a/hoodie-client/src/main/java/com/uber/hoodie/client/utils/ClientUtils.java +++ b/hudi-client/src/main/java/org/apache/hudi/client/utils/ClientUtils.java @@ -16,10 +16,10 @@ * limitations under the License. */ -package com.uber.hoodie.client.utils; +package org.apache.hudi.client.utils; -import com.uber.hoodie.common.table.HoodieTableMetaClient; -import com.uber.hoodie.config.HoodieWriteConfig; +import org.apache.hudi.common.table.HoodieTableMetaClient; +import org.apache.hudi.config.HoodieWriteConfig; import org.apache.spark.api.java.JavaSparkContext; public class ClientUtils { diff --git a/hoodie-client/src/main/java/com/uber/hoodie/config/HoodieCompactionConfig.java b/hudi-client/src/main/java/org/apache/hudi/config/HoodieCompactionConfig.java similarity index 98% rename from hoodie-client/src/main/java/com/uber/hoodie/config/HoodieCompactionConfig.java rename to hudi-client/src/main/java/org/apache/hudi/config/HoodieCompactionConfig.java index fb32d88a5..0f7a5331a 100644 --- a/hoodie-client/src/main/java/com/uber/hoodie/config/HoodieCompactionConfig.java +++ b/hudi-client/src/main/java/org/apache/hudi/config/HoodieCompactionConfig.java @@ -16,18 +16,18 @@ * limitations under the License. */ -package com.uber.hoodie.config; +package org.apache.hudi.config; import com.google.common.base.Preconditions; -import com.uber.hoodie.common.model.HoodieAvroPayload; -import com.uber.hoodie.common.model.HoodieCleaningPolicy; -import com.uber.hoodie.io.compact.strategy.CompactionStrategy; -import com.uber.hoodie.io.compact.strategy.LogFileSizeBasedCompactionStrategy; import java.io.File; import java.io.FileReader; import java.io.IOException; import java.util.Properties; import javax.annotation.concurrent.Immutable; +import org.apache.hudi.common.model.HoodieAvroPayload; +import org.apache.hudi.common.model.HoodieCleaningPolicy; +import org.apache.hudi.io.compact.strategy.CompactionStrategy; +import org.apache.hudi.io.compact.strategy.LogFileSizeBasedCompactionStrategy; /** * Compaction related config diff --git a/hoodie-client/src/main/java/com/uber/hoodie/config/HoodieHBaseIndexConfig.java b/hudi-client/src/main/java/org/apache/hudi/config/HoodieHBaseIndexConfig.java similarity index 99% rename from hoodie-client/src/main/java/com/uber/hoodie/config/HoodieHBaseIndexConfig.java rename to hudi-client/src/main/java/org/apache/hudi/config/HoodieHBaseIndexConfig.java index e1acda96c..256bba920 100644 --- a/hoodie-client/src/main/java/com/uber/hoodie/config/HoodieHBaseIndexConfig.java +++ b/hudi-client/src/main/java/org/apache/hudi/config/HoodieHBaseIndexConfig.java @@ -16,13 +16,13 @@ * limitations under the License. */ -package com.uber.hoodie.config; +package org.apache.hudi.config; -import com.uber.hoodie.index.hbase.DefaultHBaseQPSResourceAllocator; import java.io.File; import java.io.FileReader; import java.io.IOException; import java.util.Properties; +import org.apache.hudi.index.hbase.DefaultHBaseQPSResourceAllocator; public class HoodieHBaseIndexConfig extends DefaultHoodieConfig { diff --git a/hoodie-client/src/main/java/com/uber/hoodie/config/HoodieIndexConfig.java b/hudi-client/src/main/java/org/apache/hudi/config/HoodieIndexConfig.java similarity index 99% rename from hoodie-client/src/main/java/com/uber/hoodie/config/HoodieIndexConfig.java rename to hudi-client/src/main/java/org/apache/hudi/config/HoodieIndexConfig.java index 9cbc734b6..85786782f 100644 --- a/hoodie-client/src/main/java/com/uber/hoodie/config/HoodieIndexConfig.java +++ b/hudi-client/src/main/java/org/apache/hudi/config/HoodieIndexConfig.java @@ -16,14 +16,14 @@ * limitations under the License. */ -package com.uber.hoodie.config; +package org.apache.hudi.config; -import com.uber.hoodie.index.HoodieIndex; import java.io.File; import java.io.FileReader; import java.io.IOException; import java.util.Properties; import javax.annotation.concurrent.Immutable; +import org.apache.hudi.index.HoodieIndex; /** * Indexing related config diff --git a/hoodie-client/src/main/java/com/uber/hoodie/config/HoodieMemoryConfig.java b/hudi-client/src/main/java/org/apache/hudi/config/HoodieMemoryConfig.java similarity index 99% rename from hoodie-client/src/main/java/com/uber/hoodie/config/HoodieMemoryConfig.java rename to hudi-client/src/main/java/org/apache/hudi/config/HoodieMemoryConfig.java index e47bff685..7a41c5745 100644 --- a/hoodie-client/src/main/java/com/uber/hoodie/config/HoodieMemoryConfig.java +++ b/hudi-client/src/main/java/org/apache/hudi/config/HoodieMemoryConfig.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package com.uber.hoodie.config; +package org.apache.hudi.config; import java.io.File; import java.io.FileReader; diff --git a/hoodie-client/src/main/java/com/uber/hoodie/config/HoodieMetricsConfig.java b/hudi-client/src/main/java/org/apache/hudi/config/HoodieMetricsConfig.java similarity index 98% rename from hoodie-client/src/main/java/com/uber/hoodie/config/HoodieMetricsConfig.java rename to hudi-client/src/main/java/org/apache/hudi/config/HoodieMetricsConfig.java index cdf2a12c2..e21e00288 100644 --- a/hoodie-client/src/main/java/com/uber/hoodie/config/HoodieMetricsConfig.java +++ b/hudi-client/src/main/java/org/apache/hudi/config/HoodieMetricsConfig.java @@ -16,14 +16,14 @@ * limitations under the License. */ -package com.uber.hoodie.config; +package org.apache.hudi.config; -import com.uber.hoodie.metrics.MetricsReporterType; import java.io.File; import java.io.FileReader; import java.io.IOException; import java.util.Properties; import javax.annotation.concurrent.Immutable; +import org.apache.hudi.metrics.MetricsReporterType; /** * Fetch the configurations used by the Metrics system. diff --git a/hoodie-client/src/main/java/com/uber/hoodie/config/HoodieStorageConfig.java b/hudi-client/src/main/java/org/apache/hudi/config/HoodieStorageConfig.java similarity index 99% rename from hoodie-client/src/main/java/com/uber/hoodie/config/HoodieStorageConfig.java rename to hudi-client/src/main/java/org/apache/hudi/config/HoodieStorageConfig.java index 27c0e39d7..27060d5e4 100644 --- a/hoodie-client/src/main/java/com/uber/hoodie/config/HoodieStorageConfig.java +++ b/hudi-client/src/main/java/org/apache/hudi/config/HoodieStorageConfig.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package com.uber.hoodie.config; +package org.apache.hudi.config; import java.io.File; import java.io.FileReader; diff --git a/hoodie-client/src/main/java/com/uber/hoodie/config/HoodieWriteConfig.java b/hudi-client/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java similarity index 98% rename from hoodie-client/src/main/java/com/uber/hoodie/config/HoodieWriteConfig.java rename to hudi-client/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java index 63ba10ef8..82a8f9055 100644 --- a/hoodie-client/src/main/java/com/uber/hoodie/config/HoodieWriteConfig.java +++ b/hudi-client/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java @@ -16,17 +16,9 @@ * limitations under the License. */ -package com.uber.hoodie.config; +package org.apache.hudi.config; import com.google.common.base.Preconditions; -import com.uber.hoodie.WriteStatus; -import com.uber.hoodie.common.model.HoodieCleaningPolicy; -import com.uber.hoodie.common.table.view.FileSystemViewStorageConfig; -import com.uber.hoodie.common.util.ConsistencyGuardConfig; -import com.uber.hoodie.common.util.ReflectionUtils; -import com.uber.hoodie.index.HoodieIndex; -import com.uber.hoodie.io.compact.strategy.CompactionStrategy; -import com.uber.hoodie.metrics.MetricsReporterType; import java.io.File; import java.io.FileReader; import java.io.IOException; @@ -34,11 +26,20 @@ import java.io.InputStream; import java.util.Map; import java.util.Properties; import javax.annotation.concurrent.Immutable; +import org.apache.hudi.HoodieWriteClient; +import org.apache.hudi.WriteStatus; +import org.apache.hudi.common.model.HoodieCleaningPolicy; +import org.apache.hudi.common.table.view.FileSystemViewStorageConfig; +import org.apache.hudi.common.util.ConsistencyGuardConfig; +import org.apache.hudi.common.util.ReflectionUtils; +import org.apache.hudi.index.HoodieIndex; +import org.apache.hudi.io.compact.strategy.CompactionStrategy; +import org.apache.hudi.metrics.MetricsReporterType; import org.apache.parquet.hadoop.metadata.CompressionCodecName; import org.apache.spark.storage.StorageLevel; /** - * Class storing configs for the {@link com.uber.hoodie.HoodieWriteClient} + * Class storing configs for the {@link HoodieWriteClient} */ @Immutable public class HoodieWriteConfig extends DefaultHoodieConfig { diff --git a/hoodie-client/src/main/java/com/uber/hoodie/exception/HoodieAppendException.java b/hudi-client/src/main/java/org/apache/hudi/exception/HoodieAppendException.java similarity index 97% rename from hoodie-client/src/main/java/com/uber/hoodie/exception/HoodieAppendException.java rename to hudi-client/src/main/java/org/apache/hudi/exception/HoodieAppendException.java index 9b1e859c9..b2bb38eda 100644 --- a/hoodie-client/src/main/java/com/uber/hoodie/exception/HoodieAppendException.java +++ b/hudi-client/src/main/java/org/apache/hudi/exception/HoodieAppendException.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package com.uber.hoodie.exception; +package org.apache.hudi.exception; /** *

Exception thrown for any higher level errors when HoodieClient is doing a delta diff --git a/hoodie-client/src/main/java/com/uber/hoodie/exception/HoodieCommitException.java b/hudi-client/src/main/java/org/apache/hudi/exception/HoodieCommitException.java similarity index 97% rename from hoodie-client/src/main/java/com/uber/hoodie/exception/HoodieCommitException.java rename to hudi-client/src/main/java/org/apache/hudi/exception/HoodieCommitException.java index 410416307..46e573b80 100644 --- a/hoodie-client/src/main/java/com/uber/hoodie/exception/HoodieCommitException.java +++ b/hudi-client/src/main/java/org/apache/hudi/exception/HoodieCommitException.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package com.uber.hoodie.exception; +package org.apache.hudi.exception; /** *

Exception thrown for any higher level errors when HoodieClient is doing a Commit diff --git a/hoodie-client/src/main/java/com/uber/hoodie/exception/HoodieCompactionException.java b/hudi-client/src/main/java/org/apache/hudi/exception/HoodieCompactionException.java similarity index 96% rename from hoodie-client/src/main/java/com/uber/hoodie/exception/HoodieCompactionException.java rename to hudi-client/src/main/java/org/apache/hudi/exception/HoodieCompactionException.java index 89d805019..48bd8773b 100644 --- a/hoodie-client/src/main/java/com/uber/hoodie/exception/HoodieCompactionException.java +++ b/hudi-client/src/main/java/org/apache/hudi/exception/HoodieCompactionException.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package com.uber.hoodie.exception; +package org.apache.hudi.exception; public class HoodieCompactionException extends HoodieException { diff --git a/hoodie-client/src/main/java/com/uber/hoodie/exception/HoodieDependentSystemUnavailableException.java b/hudi-client/src/main/java/org/apache/hudi/exception/HoodieDependentSystemUnavailableException.java similarity index 97% rename from hoodie-client/src/main/java/com/uber/hoodie/exception/HoodieDependentSystemUnavailableException.java rename to hudi-client/src/main/java/org/apache/hudi/exception/HoodieDependentSystemUnavailableException.java index cadc94e15..ffb125533 100644 --- a/hoodie-client/src/main/java/com/uber/hoodie/exception/HoodieDependentSystemUnavailableException.java +++ b/hudi-client/src/main/java/org/apache/hudi/exception/HoodieDependentSystemUnavailableException.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package com.uber.hoodie.exception; +package org.apache.hudi.exception; /** diff --git a/hoodie-client/src/main/java/com/uber/hoodie/exception/HoodieInsertException.java b/hudi-client/src/main/java/org/apache/hudi/exception/HoodieInsertException.java similarity index 96% rename from hoodie-client/src/main/java/com/uber/hoodie/exception/HoodieInsertException.java rename to hudi-client/src/main/java/org/apache/hudi/exception/HoodieInsertException.java index f3045717f..3de9fe6b1 100644 --- a/hoodie-client/src/main/java/com/uber/hoodie/exception/HoodieInsertException.java +++ b/hudi-client/src/main/java/org/apache/hudi/exception/HoodieInsertException.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package com.uber.hoodie.exception; +package org.apache.hudi.exception; /** *

Exception thrown for any higher level errors when HoodieClient is doing a bulk diff --git a/hoodie-client/src/main/java/com/uber/hoodie/exception/HoodieRollbackException.java b/hudi-client/src/main/java/org/apache/hudi/exception/HoodieRollbackException.java similarity index 96% rename from hoodie-client/src/main/java/com/uber/hoodie/exception/HoodieRollbackException.java rename to hudi-client/src/main/java/org/apache/hudi/exception/HoodieRollbackException.java index bcbe2e319..d7683e090 100644 --- a/hoodie-client/src/main/java/com/uber/hoodie/exception/HoodieRollbackException.java +++ b/hudi-client/src/main/java/org/apache/hudi/exception/HoodieRollbackException.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package com.uber.hoodie.exception; +package org.apache.hudi.exception; public class HoodieRollbackException extends HoodieException { diff --git a/hoodie-client/src/main/java/com/uber/hoodie/exception/HoodieSavepointException.java b/hudi-client/src/main/java/org/apache/hudi/exception/HoodieSavepointException.java similarity index 96% rename from hoodie-client/src/main/java/com/uber/hoodie/exception/HoodieSavepointException.java rename to hudi-client/src/main/java/org/apache/hudi/exception/HoodieSavepointException.java index 55066c040..4a75f0bbc 100644 --- a/hoodie-client/src/main/java/com/uber/hoodie/exception/HoodieSavepointException.java +++ b/hudi-client/src/main/java/org/apache/hudi/exception/HoodieSavepointException.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package com.uber.hoodie.exception; +package org.apache.hudi.exception; public class HoodieSavepointException extends HoodieException { diff --git a/hoodie-client/src/main/java/com/uber/hoodie/exception/HoodieUpsertException.java b/hudi-client/src/main/java/org/apache/hudi/exception/HoodieUpsertException.java similarity index 97% rename from hoodie-client/src/main/java/com/uber/hoodie/exception/HoodieUpsertException.java rename to hudi-client/src/main/java/org/apache/hudi/exception/HoodieUpsertException.java index a11a89789..c57dd697c 100644 --- a/hoodie-client/src/main/java/com/uber/hoodie/exception/HoodieUpsertException.java +++ b/hudi-client/src/main/java/org/apache/hudi/exception/HoodieUpsertException.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package com.uber.hoodie.exception; +package org.apache.hudi.exception; /** *

Exception thrown for any higher level errors when HoodieClient is doing a diff --git a/hoodie-client/src/main/java/com/uber/hoodie/func/BulkInsertMapFunction.java b/hudi-client/src/main/java/org/apache/hudi/func/BulkInsertMapFunction.java similarity index 87% rename from hoodie-client/src/main/java/com/uber/hoodie/func/BulkInsertMapFunction.java rename to hudi-client/src/main/java/org/apache/hudi/func/BulkInsertMapFunction.java index 6fe40900b..86a2a9fb7 100644 --- a/hoodie-client/src/main/java/com/uber/hoodie/func/BulkInsertMapFunction.java +++ b/hudi-client/src/main/java/org/apache/hudi/func/BulkInsertMapFunction.java @@ -16,15 +16,15 @@ * limitations under the License. */ -package com.uber.hoodie.func; +package org.apache.hudi.func; -import com.uber.hoodie.WriteStatus; -import com.uber.hoodie.common.model.HoodieRecord; -import com.uber.hoodie.common.model.HoodieRecordPayload; -import com.uber.hoodie.config.HoodieWriteConfig; -import com.uber.hoodie.table.HoodieTable; import java.util.Iterator; import java.util.List; +import org.apache.hudi.WriteStatus; +import org.apache.hudi.common.model.HoodieRecord; +import org.apache.hudi.common.model.HoodieRecordPayload; +import org.apache.hudi.config.HoodieWriteConfig; +import org.apache.hudi.table.HoodieTable; import org.apache.spark.api.java.function.Function2; diff --git a/hoodie-client/src/main/java/com/uber/hoodie/func/CopyOnWriteLazyInsertIterable.java b/hudi-client/src/main/java/org/apache/hudi/func/CopyOnWriteLazyInsertIterable.java similarity index 89% rename from hoodie-client/src/main/java/com/uber/hoodie/func/CopyOnWriteLazyInsertIterable.java rename to hudi-client/src/main/java/org/apache/hudi/func/CopyOnWriteLazyInsertIterable.java index ac4ddd7d8..a772ae6c0 100644 --- a/hoodie-client/src/main/java/com/uber/hoodie/func/CopyOnWriteLazyInsertIterable.java +++ b/hudi-client/src/main/java/org/apache/hudi/func/CopyOnWriteLazyInsertIterable.java @@ -16,25 +16,25 @@ * limitations under the License. */ -package com.uber.hoodie.func; +package org.apache.hudi.func; -import com.uber.hoodie.WriteStatus; -import com.uber.hoodie.common.model.HoodieRecord; -import com.uber.hoodie.common.model.HoodieRecordPayload; -import com.uber.hoodie.common.util.Option; -import com.uber.hoodie.common.util.queue.BoundedInMemoryExecutor; -import com.uber.hoodie.common.util.queue.BoundedInMemoryQueueConsumer; -import com.uber.hoodie.config.HoodieWriteConfig; -import com.uber.hoodie.exception.HoodieException; -import com.uber.hoodie.io.HoodieCreateHandle; -import com.uber.hoodie.io.HoodieWriteHandle; -import com.uber.hoodie.table.HoodieTable; import java.util.ArrayList; import java.util.Iterator; import java.util.List; import java.util.function.Function; import org.apache.avro.Schema; import org.apache.avro.generic.IndexedRecord; +import org.apache.hudi.WriteStatus; +import org.apache.hudi.common.model.HoodieRecord; +import org.apache.hudi.common.model.HoodieRecordPayload; +import org.apache.hudi.common.util.Option; +import org.apache.hudi.common.util.queue.BoundedInMemoryExecutor; +import org.apache.hudi.common.util.queue.BoundedInMemoryQueueConsumer; +import org.apache.hudi.config.HoodieWriteConfig; +import org.apache.hudi.exception.HoodieException; +import org.apache.hudi.io.HoodieCreateHandle; +import org.apache.hudi.io.HoodieWriteHandle; +import org.apache.hudi.table.HoodieTable; /** * Lazy Iterable, that writes a stream of HoodieRecords sorted by the partitionPath, into new @@ -93,7 +93,7 @@ public class CopyOnWriteLazyInsertIterable extend protected List computeNext() { // Executor service used for launching writer thread. BoundedInMemoryExecutor, - HoodieInsertValueGenResult, List> bufferedIteratorExecutor = null; + HoodieInsertValueGenResult, List> bufferedIteratorExecutor = null; try { final Schema schema = new Schema.Parser().parse(hoodieConfig.getSchema()); bufferedIteratorExecutor = diff --git a/hoodie-client/src/main/java/com/uber/hoodie/func/LazyIterableIterator.java b/hudi-client/src/main/java/org/apache/hudi/func/LazyIterableIterator.java similarity index 99% rename from hoodie-client/src/main/java/com/uber/hoodie/func/LazyIterableIterator.java rename to hudi-client/src/main/java/org/apache/hudi/func/LazyIterableIterator.java index 4dcbe9083..a66fd3553 100644 --- a/hoodie-client/src/main/java/com/uber/hoodie/func/LazyIterableIterator.java +++ b/hudi-client/src/main/java/org/apache/hudi/func/LazyIterableIterator.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package com.uber.hoodie.func; +package org.apache.hudi.func; import java.util.Iterator; diff --git a/hoodie-client/src/main/java/com/uber/hoodie/func/MergeOnReadLazyInsertIterable.java b/hudi-client/src/main/java/org/apache/hudi/func/MergeOnReadLazyInsertIterable.java similarity index 89% rename from hoodie-client/src/main/java/com/uber/hoodie/func/MergeOnReadLazyInsertIterable.java rename to hudi-client/src/main/java/org/apache/hudi/func/MergeOnReadLazyInsertIterable.java index e60301cef..db44bca90 100644 --- a/hoodie-client/src/main/java/com/uber/hoodie/func/MergeOnReadLazyInsertIterable.java +++ b/hudi-client/src/main/java/org/apache/hudi/func/MergeOnReadLazyInsertIterable.java @@ -16,17 +16,17 @@ * limitations under the License. */ -package com.uber.hoodie.func; +package org.apache.hudi.func; -import com.uber.hoodie.WriteStatus; -import com.uber.hoodie.common.model.HoodieRecord; -import com.uber.hoodie.common.model.HoodieRecordPayload; -import com.uber.hoodie.config.HoodieWriteConfig; -import com.uber.hoodie.io.HoodieAppendHandle; -import com.uber.hoodie.table.HoodieTable; import java.util.ArrayList; import java.util.Iterator; import java.util.List; +import org.apache.hudi.WriteStatus; +import org.apache.hudi.common.model.HoodieRecord; +import org.apache.hudi.common.model.HoodieRecordPayload; +import org.apache.hudi.config.HoodieWriteConfig; +import org.apache.hudi.io.HoodieAppendHandle; +import org.apache.hudi.table.HoodieTable; /** * Lazy Iterable, that writes a stream of HoodieRecords sorted by the partitionPath, into new diff --git a/hoodie-client/src/main/java/com/uber/hoodie/func/OperationResult.java b/hudi-client/src/main/java/org/apache/hudi/func/OperationResult.java similarity index 96% rename from hoodie-client/src/main/java/com/uber/hoodie/func/OperationResult.java rename to hudi-client/src/main/java/org/apache/hudi/func/OperationResult.java index dbb893f00..980d555b6 100644 --- a/hoodie-client/src/main/java/com/uber/hoodie/func/OperationResult.java +++ b/hudi-client/src/main/java/org/apache/hudi/func/OperationResult.java @@ -16,10 +16,10 @@ * limitations under the License. */ -package com.uber.hoodie.func; +package org.apache.hudi.func; -import com.uber.hoodie.common.util.Option; import java.io.Serializable; +import org.apache.hudi.common.util.Option; /** * Holds Operation result. Used as a result container for Compaction Admin Client (running as part of Spark-launcher diff --git a/hoodie-client/src/main/java/com/uber/hoodie/func/ParquetReaderIterator.java b/hudi-client/src/main/java/org/apache/hudi/func/ParquetReaderIterator.java similarity index 94% rename from hoodie-client/src/main/java/com/uber/hoodie/func/ParquetReaderIterator.java rename to hudi-client/src/main/java/org/apache/hudi/func/ParquetReaderIterator.java index b4ca88b9b..00c8636e1 100644 --- a/hoodie-client/src/main/java/com/uber/hoodie/func/ParquetReaderIterator.java +++ b/hudi-client/src/main/java/org/apache/hudi/func/ParquetReaderIterator.java @@ -16,12 +16,12 @@ * limitations under the License. */ -package com.uber.hoodie.func; +package org.apache.hudi.func; -import com.uber.hoodie.common.util.queue.BoundedInMemoryQueue; -import com.uber.hoodie.exception.HoodieIOException; import java.io.IOException; import java.util.Iterator; +import org.apache.hudi.common.util.queue.BoundedInMemoryQueue; +import org.apache.hudi.exception.HoodieIOException; import org.apache.parquet.hadoop.ParquetReader; /** diff --git a/hoodie-client/src/main/java/com/uber/hoodie/func/SparkBoundedInMemoryExecutor.java b/hudi-client/src/main/java/org/apache/hudi/func/SparkBoundedInMemoryExecutor.java similarity index 85% rename from hoodie-client/src/main/java/com/uber/hoodie/func/SparkBoundedInMemoryExecutor.java rename to hudi-client/src/main/java/org/apache/hudi/func/SparkBoundedInMemoryExecutor.java index 47774bff5..3924fa2c1 100644 --- a/hoodie-client/src/main/java/com/uber/hoodie/func/SparkBoundedInMemoryExecutor.java +++ b/hudi-client/src/main/java/org/apache/hudi/func/SparkBoundedInMemoryExecutor.java @@ -16,16 +16,16 @@ * limitations under the License. */ -package com.uber.hoodie.func; +package org.apache.hudi.func; -import com.uber.hoodie.common.util.Option; -import com.uber.hoodie.common.util.queue.BoundedInMemoryExecutor; -import com.uber.hoodie.common.util.queue.BoundedInMemoryQueueConsumer; -import com.uber.hoodie.common.util.queue.BoundedInMemoryQueueProducer; -import com.uber.hoodie.common.util.queue.IteratorBasedQueueProducer; -import com.uber.hoodie.config.HoodieWriteConfig; import java.util.Iterator; import java.util.function.Function; +import org.apache.hudi.common.util.Option; +import org.apache.hudi.common.util.queue.BoundedInMemoryExecutor; +import org.apache.hudi.common.util.queue.BoundedInMemoryQueueConsumer; +import org.apache.hudi.common.util.queue.BoundedInMemoryQueueProducer; +import org.apache.hudi.common.util.queue.IteratorBasedQueueProducer; +import org.apache.hudi.config.HoodieWriteConfig; import org.apache.spark.TaskContext; import org.apache.spark.TaskContext$; diff --git a/hoodie-client/src/main/java/com/uber/hoodie/index/HoodieIndex.java b/hudi-client/src/main/java/org/apache/hudi/index/HoodieIndex.java similarity index 85% rename from hoodie-client/src/main/java/com/uber/hoodie/index/HoodieIndex.java rename to hudi-client/src/main/java/org/apache/hudi/index/HoodieIndex.java index 881de3657..76b51e8ea 100644 --- a/hoodie-client/src/main/java/com/uber/hoodie/index/HoodieIndex.java +++ b/hudi-client/src/main/java/org/apache/hudi/index/HoodieIndex.java @@ -16,21 +16,22 @@ * limitations under the License. */ -package com.uber.hoodie.index; +package org.apache.hudi.index; -import com.uber.hoodie.WriteStatus; -import com.uber.hoodie.common.model.HoodieKey; -import com.uber.hoodie.common.model.HoodieRecord; -import com.uber.hoodie.common.model.HoodieRecordPayload; -import com.uber.hoodie.common.util.Option; -import com.uber.hoodie.common.util.collection.Pair; -import com.uber.hoodie.config.HoodieWriteConfig; -import com.uber.hoodie.exception.HoodieIndexException; -import com.uber.hoodie.index.bloom.HoodieBloomIndex; -import com.uber.hoodie.index.bloom.HoodieGlobalBloomIndex; -import com.uber.hoodie.index.hbase.HBaseIndex; -import com.uber.hoodie.table.HoodieTable; import java.io.Serializable; +import org.apache.hudi.WriteStatus; +import org.apache.hudi.common.model.FileSlice; +import org.apache.hudi.common.model.HoodieKey; +import org.apache.hudi.common.model.HoodieRecord; +import org.apache.hudi.common.model.HoodieRecordPayload; +import org.apache.hudi.common.util.Option; +import org.apache.hudi.common.util.collection.Pair; +import org.apache.hudi.config.HoodieWriteConfig; +import org.apache.hudi.exception.HoodieIndexException; +import org.apache.hudi.index.bloom.HoodieBloomIndex; +import org.apache.hudi.index.bloom.HoodieGlobalBloomIndex; +import org.apache.hudi.index.hbase.HBaseIndex; +import org.apache.hudi.table.HoodieTable; import org.apache.spark.api.java.JavaPairRDD; import org.apache.spark.api.java.JavaRDD; import org.apache.spark.api.java.JavaSparkContext; @@ -102,7 +103,7 @@ public abstract class HoodieIndex implements Seri /** * This is used by storage to determine, if its safe to send inserts, straight to the log, i.e - * having a {@link com.uber.hoodie.common.model.FileSlice}, with no data file. + * having a {@link FileSlice}, with no data file. * * @return Returns true/false depending on whether the impl has this capability */ diff --git a/hoodie-client/src/main/java/com/uber/hoodie/index/InMemoryHashIndex.java b/hudi-client/src/main/java/org/apache/hudi/index/InMemoryHashIndex.java similarity index 90% rename from hoodie-client/src/main/java/com/uber/hoodie/index/InMemoryHashIndex.java rename to hudi-client/src/main/java/org/apache/hudi/index/InMemoryHashIndex.java index 4fef0add0..0ef81cf8f 100644 --- a/hoodie-client/src/main/java/com/uber/hoodie/index/InMemoryHashIndex.java +++ b/hudi-client/src/main/java/org/apache/hudi/index/InMemoryHashIndex.java @@ -16,22 +16,22 @@ * limitations under the License. */ -package com.uber.hoodie.index; +package org.apache.hudi.index; -import com.uber.hoodie.WriteStatus; -import com.uber.hoodie.common.model.HoodieKey; -import com.uber.hoodie.common.model.HoodieRecord; -import com.uber.hoodie.common.model.HoodieRecordLocation; -import com.uber.hoodie.common.model.HoodieRecordPayload; -import com.uber.hoodie.common.util.Option; -import com.uber.hoodie.common.util.collection.Pair; -import com.uber.hoodie.config.HoodieWriteConfig; -import com.uber.hoodie.table.HoodieTable; import java.util.ArrayList; import java.util.Iterator; import java.util.List; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; +import org.apache.hudi.WriteStatus; +import org.apache.hudi.common.model.HoodieKey; +import org.apache.hudi.common.model.HoodieRecord; +import org.apache.hudi.common.model.HoodieRecordLocation; +import org.apache.hudi.common.model.HoodieRecordPayload; +import org.apache.hudi.common.util.Option; +import org.apache.hudi.common.util.collection.Pair; +import org.apache.hudi.config.HoodieWriteConfig; +import org.apache.hudi.table.HoodieTable; import org.apache.spark.api.java.JavaPairRDD; import org.apache.spark.api.java.JavaRDD; import org.apache.spark.api.java.JavaSparkContext; diff --git a/hoodie-client/src/main/java/com/uber/hoodie/index/bloom/BloomIndexFileInfo.java b/hudi-client/src/main/java/org/apache/hudi/index/bloom/BloomIndexFileInfo.java similarity index 98% rename from hoodie-client/src/main/java/com/uber/hoodie/index/bloom/BloomIndexFileInfo.java rename to hudi-client/src/main/java/org/apache/hudi/index/bloom/BloomIndexFileInfo.java index 2a56d8a25..007c1af41 100644 --- a/hoodie-client/src/main/java/com/uber/hoodie/index/bloom/BloomIndexFileInfo.java +++ b/hudi-client/src/main/java/org/apache/hudi/index/bloom/BloomIndexFileInfo.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package com.uber.hoodie.index.bloom; +package org.apache.hudi.index.bloom; import com.google.common.base.Objects; import java.io.Serializable; diff --git a/hoodie-client/src/main/java/com/uber/hoodie/index/bloom/BucketizedBloomCheckPartitioner.java b/hudi-client/src/main/java/org/apache/hudi/index/bloom/BucketizedBloomCheckPartitioner.java similarity index 98% rename from hoodie-client/src/main/java/com/uber/hoodie/index/bloom/BucketizedBloomCheckPartitioner.java rename to hudi-client/src/main/java/org/apache/hudi/index/bloom/BucketizedBloomCheckPartitioner.java index a360777ff..f1150f046 100644 --- a/hoodie-client/src/main/java/com/uber/hoodie/index/bloom/BucketizedBloomCheckPartitioner.java +++ b/hudi-client/src/main/java/org/apache/hudi/index/bloom/BucketizedBloomCheckPartitioner.java @@ -16,18 +16,17 @@ * limitations under the License. */ -package com.uber.hoodie.index.bloom; +package org.apache.hudi.index.bloom; import com.google.common.annotations.VisibleForTesting; import com.google.common.hash.Hashing; -import com.uber.hoodie.common.util.collection.Pair; import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.concurrent.atomic.AtomicInteger; - +import org.apache.hudi.common.util.collection.Pair; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; import org.apache.spark.Partitioner; diff --git a/hoodie-client/src/main/java/com/uber/hoodie/index/bloom/HoodieBloomIndex.java b/hudi-client/src/main/java/org/apache/hudi/index/bloom/HoodieBloomIndex.java similarity index 95% rename from hoodie-client/src/main/java/com/uber/hoodie/index/bloom/HoodieBloomIndex.java rename to hudi-client/src/main/java/org/apache/hudi/index/bloom/HoodieBloomIndex.java index 7e9d76dc5..133f1e4a3 100644 --- a/hoodie-client/src/main/java/com/uber/hoodie/index/bloom/HoodieBloomIndex.java +++ b/hudi-client/src/main/java/org/apache/hudi/index/bloom/HoodieBloomIndex.java @@ -16,31 +16,32 @@ * limitations under the License. */ -package com.uber.hoodie.index.bloom; +package org.apache.hudi.index.bloom; import static java.util.stream.Collectors.groupingBy; import static java.util.stream.Collectors.mapping; import static java.util.stream.Collectors.toList; import com.google.common.annotations.VisibleForTesting; -import com.uber.hoodie.WriteStatus; -import com.uber.hoodie.common.model.HoodieKey; -import com.uber.hoodie.common.model.HoodieRecord; -import com.uber.hoodie.common.model.HoodieRecordLocation; -import com.uber.hoodie.common.model.HoodieRecordPayload; -import com.uber.hoodie.common.table.timeline.HoodieInstant; -import com.uber.hoodie.common.util.Option; -import com.uber.hoodie.common.util.collection.Pair; -import com.uber.hoodie.config.HoodieWriteConfig; -import com.uber.hoodie.exception.MetadataNotFoundException; -import com.uber.hoodie.index.HoodieIndex; -import com.uber.hoodie.io.HoodieRangeInfoHandle; -import com.uber.hoodie.table.HoodieTable; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.stream.Collectors; +import org.apache.hudi.WriteStatus; +import org.apache.hudi.common.model.HoodieKey; +import org.apache.hudi.common.model.HoodieRecord; +import org.apache.hudi.common.model.HoodieRecordLocation; +import org.apache.hudi.common.model.HoodieRecordPayload; +import org.apache.hudi.common.table.timeline.HoodieInstant; +import org.apache.hudi.common.util.Option; +import org.apache.hudi.common.util.collection.Pair; +import org.apache.hudi.config.HoodieIndexConfig; +import org.apache.hudi.config.HoodieWriteConfig; +import org.apache.hudi.exception.MetadataNotFoundException; +import org.apache.hudi.index.HoodieIndex; +import org.apache.hudi.io.HoodieRangeInfoHandle; +import org.apache.hudi.table.HoodieTable; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; import org.apache.spark.Partitioner; @@ -192,7 +193,7 @@ public class HoodieBloomIndex extends HoodieIndex * Compute the minimum parallelism needed to play well with the spark 2GB limitation.. The index lookup can be skewed * in three dimensions : #files, #partitions, #records

To be able to smoothly handle skews, we need to compute how * to split each partitions into subpartitions. We do it here, in a way that keeps the amount of each Spark join - * partition to < 2GB.

If {@link com.uber.hoodie.config.HoodieIndexConfig#BLOOM_INDEX_PARALLELISM_PROP} is + * partition to < 2GB.

If {@link HoodieIndexConfig#BLOOM_INDEX_PARALLELISM_PROP} is * specified as a NON-zero number, then that is used explicitly. */ int computeSafeParallelism(Map recordsPerPartition, Map comparisonsPerFileGroup) { diff --git a/hoodie-client/src/main/java/com/uber/hoodie/index/bloom/HoodieBloomIndexCheckFunction.java b/hudi-client/src/main/java/org/apache/hudi/index/bloom/HoodieBloomIndexCheckFunction.java similarity index 88% rename from hoodie-client/src/main/java/com/uber/hoodie/index/bloom/HoodieBloomIndexCheckFunction.java rename to hudi-client/src/main/java/org/apache/hudi/index/bloom/HoodieBloomIndexCheckFunction.java index 3dd3df5cd..8fa2103e1 100644 --- a/hoodie-client/src/main/java/com/uber/hoodie/index/bloom/HoodieBloomIndexCheckFunction.java +++ b/hudi-client/src/main/java/org/apache/hudi/index/bloom/HoodieBloomIndexCheckFunction.java @@ -16,20 +16,20 @@ * limitations under the License. */ -package com.uber.hoodie.index.bloom; +package org.apache.hudi.index.bloom; -import com.uber.hoodie.common.model.HoodieKey; -import com.uber.hoodie.common.util.collection.Pair; -import com.uber.hoodie.config.HoodieWriteConfig; -import com.uber.hoodie.exception.HoodieException; -import com.uber.hoodie.exception.HoodieIndexException; -import com.uber.hoodie.func.LazyIterableIterator; -import com.uber.hoodie.io.HoodieKeyLookupHandle; -import com.uber.hoodie.io.HoodieKeyLookupHandle.KeyLookupResult; -import com.uber.hoodie.table.HoodieTable; import java.util.ArrayList; import java.util.Iterator; import java.util.List; +import org.apache.hudi.common.model.HoodieKey; +import org.apache.hudi.common.util.collection.Pair; +import org.apache.hudi.config.HoodieWriteConfig; +import org.apache.hudi.exception.HoodieException; +import org.apache.hudi.exception.HoodieIndexException; +import org.apache.hudi.func.LazyIterableIterator; +import org.apache.hudi.io.HoodieKeyLookupHandle; +import org.apache.hudi.io.HoodieKeyLookupHandle.KeyLookupResult; +import org.apache.hudi.table.HoodieTable; import org.apache.spark.api.java.function.Function2; import scala.Tuple2; diff --git a/hoodie-client/src/main/java/com/uber/hoodie/index/bloom/HoodieGlobalBloomIndex.java b/hudi-client/src/main/java/org/apache/hudi/index/bloom/HoodieGlobalBloomIndex.java similarity index 90% rename from hoodie-client/src/main/java/com/uber/hoodie/index/bloom/HoodieGlobalBloomIndex.java rename to hudi-client/src/main/java/org/apache/hudi/index/bloom/HoodieGlobalBloomIndex.java index 188502a1b..6dc91562c 100644 --- a/hoodie-client/src/main/java/com/uber/hoodie/index/bloom/HoodieGlobalBloomIndex.java +++ b/hudi-client/src/main/java/org/apache/hudi/index/bloom/HoodieGlobalBloomIndex.java @@ -16,25 +16,25 @@ * limitations under the License. */ -package com.uber.hoodie.index.bloom; +package org.apache.hudi.index.bloom; import com.google.common.annotations.VisibleForTesting; -import com.uber.hoodie.common.model.HoodieKey; -import com.uber.hoodie.common.model.HoodieRecord; -import com.uber.hoodie.common.model.HoodieRecordLocation; -import com.uber.hoodie.common.model.HoodieRecordPayload; -import com.uber.hoodie.common.table.HoodieTableMetaClient; -import com.uber.hoodie.common.util.FSUtils; -import com.uber.hoodie.common.util.Option; -import com.uber.hoodie.config.HoodieWriteConfig; -import com.uber.hoodie.exception.HoodieIOException; -import com.uber.hoodie.table.HoodieTable; import java.io.IOException; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Map.Entry; import java.util.stream.Collectors; +import org.apache.hudi.common.model.HoodieKey; +import org.apache.hudi.common.model.HoodieRecord; +import org.apache.hudi.common.model.HoodieRecordLocation; +import org.apache.hudi.common.model.HoodieRecordPayload; +import org.apache.hudi.common.table.HoodieTableMetaClient; +import org.apache.hudi.common.util.FSUtils; +import org.apache.hudi.common.util.Option; +import org.apache.hudi.config.HoodieWriteConfig; +import org.apache.hudi.exception.HoodieIOException; +import org.apache.hudi.table.HoodieTable; import org.apache.spark.api.java.JavaPairRDD; import org.apache.spark.api.java.JavaRDD; import org.apache.spark.api.java.JavaSparkContext; diff --git a/hoodie-client/src/main/java/com/uber/hoodie/index/bloom/IndexFileFilter.java b/hudi-client/src/main/java/org/apache/hudi/index/bloom/IndexFileFilter.java similarity index 97% rename from hoodie-client/src/main/java/com/uber/hoodie/index/bloom/IndexFileFilter.java rename to hudi-client/src/main/java/org/apache/hudi/index/bloom/IndexFileFilter.java index 6fd1bded0..78173ff2e 100644 --- a/hoodie-client/src/main/java/com/uber/hoodie/index/bloom/IndexFileFilter.java +++ b/hudi-client/src/main/java/org/apache/hudi/index/bloom/IndexFileFilter.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package com.uber.hoodie.index.bloom; +package org.apache.hudi.index.bloom; import java.io.Serializable; import java.util.Set; diff --git a/hoodie-client/src/main/java/com/uber/hoodie/index/bloom/IntervalTreeBasedGlobalIndexFileFilter.java b/hudi-client/src/main/java/org/apache/hudi/index/bloom/IntervalTreeBasedGlobalIndexFileFilter.java similarity index 98% rename from hoodie-client/src/main/java/com/uber/hoodie/index/bloom/IntervalTreeBasedGlobalIndexFileFilter.java rename to hudi-client/src/main/java/org/apache/hudi/index/bloom/IntervalTreeBasedGlobalIndexFileFilter.java index c675ffcec..65bb5f3c3 100644 --- a/hoodie-client/src/main/java/com/uber/hoodie/index/bloom/IntervalTreeBasedGlobalIndexFileFilter.java +++ b/hudi-client/src/main/java/org/apache/hudi/index/bloom/IntervalTreeBasedGlobalIndexFileFilter.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package com.uber.hoodie.index.bloom; +package org.apache.hudi.index.bloom; import java.util.Collection; import java.util.Collections; diff --git a/hoodie-client/src/main/java/com/uber/hoodie/index/bloom/IntervalTreeBasedIndexFileFilter.java b/hudi-client/src/main/java/org/apache/hudi/index/bloom/IntervalTreeBasedIndexFileFilter.java similarity index 98% rename from hoodie-client/src/main/java/com/uber/hoodie/index/bloom/IntervalTreeBasedIndexFileFilter.java rename to hudi-client/src/main/java/org/apache/hudi/index/bloom/IntervalTreeBasedIndexFileFilter.java index 412e778f3..29d5566c6 100644 --- a/hoodie-client/src/main/java/com/uber/hoodie/index/bloom/IntervalTreeBasedIndexFileFilter.java +++ b/hudi-client/src/main/java/org/apache/hudi/index/bloom/IntervalTreeBasedIndexFileFilter.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package com.uber.hoodie.index.bloom; +package org.apache.hudi.index.bloom; import java.util.Collections; import java.util.HashMap; diff --git a/hoodie-client/src/main/java/com/uber/hoodie/index/bloom/KeyLookupResult.java b/hudi-client/src/main/java/org/apache/hudi/index/bloom/KeyLookupResult.java similarity index 97% rename from hoodie-client/src/main/java/com/uber/hoodie/index/bloom/KeyLookupResult.java rename to hudi-client/src/main/java/org/apache/hudi/index/bloom/KeyLookupResult.java index 7b321f1f0..631ff768c 100644 --- a/hoodie-client/src/main/java/com/uber/hoodie/index/bloom/KeyLookupResult.java +++ b/hudi-client/src/main/java/org/apache/hudi/index/bloom/KeyLookupResult.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package com.uber.hoodie.index.bloom; +package org.apache.hudi.index.bloom; import java.util.List; diff --git a/hoodie-client/src/main/java/com/uber/hoodie/index/bloom/KeyRangeLookupTree.java b/hudi-client/src/main/java/org/apache/hudi/index/bloom/KeyRangeLookupTree.java similarity index 99% rename from hoodie-client/src/main/java/com/uber/hoodie/index/bloom/KeyRangeLookupTree.java rename to hudi-client/src/main/java/org/apache/hudi/index/bloom/KeyRangeLookupTree.java index 5c05f8dd5..4bf69125f 100644 --- a/hoodie-client/src/main/java/com/uber/hoodie/index/bloom/KeyRangeLookupTree.java +++ b/hudi-client/src/main/java/org/apache/hudi/index/bloom/KeyRangeLookupTree.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package com.uber.hoodie.index.bloom; +package org.apache.hudi.index.bloom; import java.io.Serializable; import java.util.HashSet; diff --git a/hoodie-client/src/main/java/com/uber/hoodie/index/bloom/KeyRangeNode.java b/hudi-client/src/main/java/org/apache/hudi/index/bloom/KeyRangeNode.java similarity index 99% rename from hoodie-client/src/main/java/com/uber/hoodie/index/bloom/KeyRangeNode.java rename to hudi-client/src/main/java/org/apache/hudi/index/bloom/KeyRangeNode.java index 96790c41d..a96bae463 100644 --- a/hoodie-client/src/main/java/com/uber/hoodie/index/bloom/KeyRangeNode.java +++ b/hudi-client/src/main/java/org/apache/hudi/index/bloom/KeyRangeNode.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package com.uber.hoodie.index.bloom; +package org.apache.hudi.index.bloom; import java.io.Serializable; import java.util.ArrayList; diff --git a/hoodie-client/src/main/java/com/uber/hoodie/index/bloom/ListBasedGlobalIndexFileFilter.java b/hudi-client/src/main/java/org/apache/hudi/index/bloom/ListBasedGlobalIndexFileFilter.java similarity index 98% rename from hoodie-client/src/main/java/com/uber/hoodie/index/bloom/ListBasedGlobalIndexFileFilter.java rename to hudi-client/src/main/java/org/apache/hudi/index/bloom/ListBasedGlobalIndexFileFilter.java index f904126f3..bccb03f36 100644 --- a/hoodie-client/src/main/java/com/uber/hoodie/index/bloom/ListBasedGlobalIndexFileFilter.java +++ b/hudi-client/src/main/java/org/apache/hudi/index/bloom/ListBasedGlobalIndexFileFilter.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package com.uber.hoodie.index.bloom; +package org.apache.hudi.index.bloom; import java.util.HashSet; import java.util.List; diff --git a/hoodie-client/src/main/java/com/uber/hoodie/index/bloom/ListBasedIndexFileFilter.java b/hudi-client/src/main/java/org/apache/hudi/index/bloom/ListBasedIndexFileFilter.java similarity index 98% rename from hoodie-client/src/main/java/com/uber/hoodie/index/bloom/ListBasedIndexFileFilter.java rename to hudi-client/src/main/java/org/apache/hudi/index/bloom/ListBasedIndexFileFilter.java index b99280b98..967320998 100644 --- a/hoodie-client/src/main/java/com/uber/hoodie/index/bloom/ListBasedIndexFileFilter.java +++ b/hudi-client/src/main/java/org/apache/hudi/index/bloom/ListBasedIndexFileFilter.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package com.uber.hoodie.index.bloom; +package org.apache.hudi.index.bloom; import java.util.HashSet; import java.util.List; diff --git a/hoodie-client/src/main/java/com/uber/hoodie/index/hbase/DefaultHBaseQPSResourceAllocator.java b/hudi-client/src/main/java/org/apache/hudi/index/hbase/DefaultHBaseQPSResourceAllocator.java similarity index 95% rename from hoodie-client/src/main/java/com/uber/hoodie/index/hbase/DefaultHBaseQPSResourceAllocator.java rename to hudi-client/src/main/java/org/apache/hudi/index/hbase/DefaultHBaseQPSResourceAllocator.java index 44822e47c..4686c1fc6 100644 --- a/hoodie-client/src/main/java/com/uber/hoodie/index/hbase/DefaultHBaseQPSResourceAllocator.java +++ b/hudi-client/src/main/java/org/apache/hudi/index/hbase/DefaultHBaseQPSResourceAllocator.java @@ -16,10 +16,9 @@ * limitations under the License. */ -package com.uber.hoodie.index.hbase; - -import com.uber.hoodie.config.HoodieWriteConfig; +package org.apache.hudi.index.hbase; +import org.apache.hudi.config.HoodieWriteConfig; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; diff --git a/hoodie-client/src/main/java/com/uber/hoodie/index/hbase/HBaseIndex.java b/hudi-client/src/main/java/org/apache/hudi/index/hbase/HBaseIndex.java similarity index 96% rename from hoodie-client/src/main/java/com/uber/hoodie/index/hbase/HBaseIndex.java rename to hudi-client/src/main/java/org/apache/hudi/index/hbase/HBaseIndex.java index 5012c49d4..8399c4f31 100644 --- a/hoodie-client/src/main/java/com/uber/hoodie/index/hbase/HBaseIndex.java +++ b/hudi-client/src/main/java/org/apache/hudi/index/hbase/HBaseIndex.java @@ -16,26 +16,9 @@ * limitations under the License. */ -package com.uber.hoodie.index.hbase; +package org.apache.hudi.index.hbase; import com.google.common.annotations.VisibleForTesting; -import com.uber.hoodie.WriteStatus; -import com.uber.hoodie.common.model.HoodieKey; -import com.uber.hoodie.common.model.HoodieRecord; -import com.uber.hoodie.common.model.HoodieRecordLocation; -import com.uber.hoodie.common.model.HoodieRecordPayload; -import com.uber.hoodie.common.table.HoodieTableMetaClient; -import com.uber.hoodie.common.table.HoodieTimeline; -import com.uber.hoodie.common.table.timeline.HoodieInstant; -import com.uber.hoodie.common.util.Option; -import com.uber.hoodie.common.util.ReflectionUtils; -import com.uber.hoodie.common.util.collection.Pair; -import com.uber.hoodie.config.HoodieIndexConfig; -import com.uber.hoodie.config.HoodieWriteConfig; -import com.uber.hoodie.exception.HoodieDependentSystemUnavailableException; -import com.uber.hoodie.exception.HoodieIndexException; -import com.uber.hoodie.index.HoodieIndex; -import com.uber.hoodie.table.HoodieTable; import java.io.IOException; import java.io.Serializable; import java.util.ArrayList; @@ -54,6 +37,23 @@ import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.client.RegionLocator; import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hudi.WriteStatus; +import org.apache.hudi.common.model.HoodieKey; +import org.apache.hudi.common.model.HoodieRecord; +import org.apache.hudi.common.model.HoodieRecordLocation; +import org.apache.hudi.common.model.HoodieRecordPayload; +import org.apache.hudi.common.table.HoodieTableMetaClient; +import org.apache.hudi.common.table.HoodieTimeline; +import org.apache.hudi.common.table.timeline.HoodieInstant; +import org.apache.hudi.common.util.Option; +import org.apache.hudi.common.util.ReflectionUtils; +import org.apache.hudi.common.util.collection.Pair; +import org.apache.hudi.config.HoodieIndexConfig; +import org.apache.hudi.config.HoodieWriteConfig; +import org.apache.hudi.exception.HoodieDependentSystemUnavailableException; +import org.apache.hudi.exception.HoodieIndexException; +import org.apache.hudi.index.HoodieIndex; +import org.apache.hudi.table.HoodieTable; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; import org.apache.spark.SparkConf; diff --git a/hoodie-client/src/main/java/com/uber/hoodie/index/hbase/HBaseIndexQPSResourceAllocator.java b/hudi-client/src/main/java/org/apache/hudi/index/hbase/HBaseIndexQPSResourceAllocator.java similarity index 98% rename from hoodie-client/src/main/java/com/uber/hoodie/index/hbase/HBaseIndexQPSResourceAllocator.java rename to hudi-client/src/main/java/org/apache/hudi/index/hbase/HBaseIndexQPSResourceAllocator.java index 8a2e3313b..0adc24738 100644 --- a/hoodie-client/src/main/java/com/uber/hoodie/index/hbase/HBaseIndexQPSResourceAllocator.java +++ b/hudi-client/src/main/java/org/apache/hudi/index/hbase/HBaseIndexQPSResourceAllocator.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package com.uber.hoodie.index.hbase; +package org.apache.hudi.index.hbase; import java.io.Serializable; diff --git a/hoodie-client/src/main/java/com/uber/hoodie/io/HoodieAppendHandle.java b/hudi-client/src/main/java/org/apache/hudi/io/HoodieAppendHandle.java similarity index 89% rename from hoodie-client/src/main/java/com/uber/hoodie/io/HoodieAppendHandle.java rename to hudi-client/src/main/java/org/apache/hudi/io/HoodieAppendHandle.java index c9906cc62..6dbcb8b19 100644 --- a/hoodie-client/src/main/java/com/uber/hoodie/io/HoodieAppendHandle.java +++ b/hudi-client/src/main/java/org/apache/hudi/io/HoodieAppendHandle.java @@ -16,32 +16,9 @@ * limitations under the License. */ -package com.uber.hoodie.io; +package org.apache.hudi.io; import com.beust.jcommander.internal.Maps; -import com.uber.hoodie.WriteStatus; -import com.uber.hoodie.common.model.FileSlice; -import com.uber.hoodie.common.model.HoodieDeltaWriteStat; -import com.uber.hoodie.common.model.HoodieKey; -import com.uber.hoodie.common.model.HoodieLogFile; -import com.uber.hoodie.common.model.HoodieRecord; -import com.uber.hoodie.common.model.HoodieRecordLocation; -import com.uber.hoodie.common.model.HoodieRecordPayload; -import com.uber.hoodie.common.model.HoodieWriteStat; -import com.uber.hoodie.common.model.HoodieWriteStat.RuntimeStats; -import com.uber.hoodie.common.table.TableFileSystemView.RealtimeView; -import com.uber.hoodie.common.table.log.HoodieLogFormat; -import com.uber.hoodie.common.table.log.HoodieLogFormat.Writer; -import com.uber.hoodie.common.table.log.block.HoodieAvroDataBlock; -import com.uber.hoodie.common.table.log.block.HoodieDeleteBlock; -import com.uber.hoodie.common.table.log.block.HoodieLogBlock; -import com.uber.hoodie.common.util.FSUtils; -import com.uber.hoodie.common.util.HoodieAvroUtils; -import com.uber.hoodie.common.util.Option; -import com.uber.hoodie.config.HoodieWriteConfig; -import com.uber.hoodie.exception.HoodieAppendException; -import com.uber.hoodie.exception.HoodieUpsertException; -import com.uber.hoodie.table.HoodieTable; import java.io.IOException; import java.util.ArrayList; import java.util.Iterator; @@ -51,6 +28,30 @@ import java.util.concurrent.atomic.AtomicLong; import org.apache.avro.generic.GenericRecord; import org.apache.avro.generic.IndexedRecord; import org.apache.hadoop.fs.Path; +import org.apache.hudi.WriteStatus; +import org.apache.hudi.common.model.FileSlice; +import org.apache.hudi.common.model.HoodieDeltaWriteStat; +import org.apache.hudi.common.model.HoodieKey; +import org.apache.hudi.common.model.HoodieLogFile; +import org.apache.hudi.common.model.HoodieRecord; +import org.apache.hudi.common.model.HoodieRecordLocation; +import org.apache.hudi.common.model.HoodieRecordPayload; +import org.apache.hudi.common.model.HoodieWriteStat; +import org.apache.hudi.common.model.HoodieWriteStat.RuntimeStats; +import org.apache.hudi.common.table.TableFileSystemView.RealtimeView; +import org.apache.hudi.common.table.log.HoodieLogFormat; +import org.apache.hudi.common.table.log.HoodieLogFormat.Writer; +import org.apache.hudi.common.table.log.block.HoodieAvroDataBlock; +import org.apache.hudi.common.table.log.block.HoodieDeleteBlock; +import org.apache.hudi.common.table.log.block.HoodieLogBlock; +import org.apache.hudi.common.table.log.block.HoodieLogBlock.HeaderMetadataType; +import org.apache.hudi.common.util.FSUtils; +import org.apache.hudi.common.util.HoodieAvroUtils; +import org.apache.hudi.common.util.Option; +import org.apache.hudi.config.HoodieWriteConfig; +import org.apache.hudi.exception.HoodieAppendException; +import org.apache.hudi.exception.HoodieUpsertException; +import org.apache.hudi.table.HoodieTable; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; import org.apache.spark.TaskContext; @@ -91,7 +92,7 @@ public class HoodieAppendHandle extends HoodieWri // Max block size to limit to for a log block private int maxBlockSize = config.getLogFileDataBlockMaxSize(); // Header metadata for a log block - private Map header = Maps.newHashMap(); + private Map header = Maps.newHashMap(); // Total number of new records inserted into the delta file private long insertRecordsWritten = 0; @@ -198,7 +199,7 @@ public class HoodieAppendHandle extends HoodieWri estimatedNumberOfBytesWritten += averageRecordSize * numberOfRecords; } - private void doAppend(Map header) { + private void doAppend(Map header) { try { header.put(HoodieLogBlock.HeaderMetadataType.INSTANT_TIME, instantTime); header.put(HoodieLogBlock.HeaderMetadataType.SCHEMA, writerSchema.toString()); diff --git a/hoodie-client/src/main/java/com/uber/hoodie/io/HoodieCleanHelper.java b/hudi-client/src/main/java/org/apache/hudi/io/HoodieCleanHelper.java similarity index 92% rename from hoodie-client/src/main/java/com/uber/hoodie/io/HoodieCleanHelper.java rename to hudi-client/src/main/java/org/apache/hudi/io/HoodieCleanHelper.java index 74a88b361..b63ea39ed 100644 --- a/hoodie-client/src/main/java/com/uber/hoodie/io/HoodieCleanHelper.java +++ b/hudi-client/src/main/java/org/apache/hudi/io/HoodieCleanHelper.java @@ -16,29 +16,30 @@ * limitations under the License. */ -package com.uber.hoodie.io; +package org.apache.hudi.io; -import com.uber.hoodie.common.model.CompactionOperation; -import com.uber.hoodie.common.model.FileSlice; -import com.uber.hoodie.common.model.HoodieCleaningPolicy; -import com.uber.hoodie.common.model.HoodieDataFile; -import com.uber.hoodie.common.model.HoodieFileGroup; -import com.uber.hoodie.common.model.HoodieFileGroupId; -import com.uber.hoodie.common.model.HoodieRecordPayload; -import com.uber.hoodie.common.model.HoodieTableType; -import com.uber.hoodie.common.table.HoodieTimeline; -import com.uber.hoodie.common.table.SyncableFileSystemView; -import com.uber.hoodie.common.table.timeline.HoodieInstant; -import com.uber.hoodie.common.util.Option; -import com.uber.hoodie.common.util.collection.Pair; -import com.uber.hoodie.config.HoodieWriteConfig; -import com.uber.hoodie.table.HoodieTable; import java.io.IOException; import java.util.ArrayList; import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.stream.Collectors; +import org.apache.hudi.common.model.CompactionOperation; +import org.apache.hudi.common.model.FileSlice; +import org.apache.hudi.common.model.HoodieCleaningPolicy; +import org.apache.hudi.common.model.HoodieCommitMetadata; +import org.apache.hudi.common.model.HoodieDataFile; +import org.apache.hudi.common.model.HoodieFileGroup; +import org.apache.hudi.common.model.HoodieFileGroupId; +import org.apache.hudi.common.model.HoodieRecordPayload; +import org.apache.hudi.common.model.HoodieTableType; +import org.apache.hudi.common.table.HoodieTimeline; +import org.apache.hudi.common.table.SyncableFileSystemView; +import org.apache.hudi.common.table.timeline.HoodieInstant; +import org.apache.hudi.common.util.Option; +import org.apache.hudi.common.util.collection.Pair; +import org.apache.hudi.config.HoodieWriteConfig; +import org.apache.hudi.table.HoodieTable; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; @@ -46,7 +47,7 @@ import org.apache.log4j.Logger; * Cleaner is responsible for garbage collecting older files in a given partition path, such that *

1) It provides sufficient time for existing queries running on older versions, to close

* 2) It bounds the growth of the files in the file system

TODO: Should all cleaning be done - * based on {@link com.uber.hoodie.common.model.HoodieCommitMetadata} + * based on {@link HoodieCommitMetadata} */ public class HoodieCleanHelper> { diff --git a/hoodie-client/src/main/java/com/uber/hoodie/io/HoodieCommitArchiveLog.java b/hudi-client/src/main/java/org/apache/hudi/io/HoodieCommitArchiveLog.java similarity index 85% rename from hoodie-client/src/main/java/com/uber/hoodie/io/HoodieCommitArchiveLog.java rename to hudi-client/src/main/java/org/apache/hudi/io/HoodieCommitArchiveLog.java index cba520168..bc8b4fe8b 100644 --- a/hoodie-client/src/main/java/com/uber/hoodie/io/HoodieCommitArchiveLog.java +++ b/hudi-client/src/main/java/org/apache/hudi/io/HoodieCommitArchiveLog.java @@ -16,39 +16,12 @@ * limitations under the License. */ -package com.uber.hoodie.io; - -import static com.uber.hoodie.common.table.HoodieTimeline.COMMIT_ACTION; -import static com.uber.hoodie.common.table.HoodieTimeline.DELTA_COMMIT_ACTION; -import static com.uber.hoodie.common.table.HoodieTimeline.LESSER_OR_EQUAL; +package org.apache.hudi.io; import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.Maps; import com.google.common.collect.Sets; -import com.uber.hoodie.avro.model.HoodieArchivedMetaEntry; -import com.uber.hoodie.avro.model.HoodieCleanMetadata; -import com.uber.hoodie.avro.model.HoodieRollbackMetadata; -import com.uber.hoodie.avro.model.HoodieSavepointMetadata; -import com.uber.hoodie.common.model.ActionType; -import com.uber.hoodie.common.model.HoodieArchivedLogFile; -import com.uber.hoodie.common.model.HoodieCommitMetadata; -import com.uber.hoodie.common.model.HoodieRollingStatMetadata; -import com.uber.hoodie.common.table.HoodieTableMetaClient; -import com.uber.hoodie.common.table.HoodieTimeline; -import com.uber.hoodie.common.table.log.HoodieLogFormat; -import com.uber.hoodie.common.table.log.block.HoodieAvroDataBlock; -import com.uber.hoodie.common.table.log.block.HoodieLogBlock; -import com.uber.hoodie.common.table.timeline.HoodieActiveTimeline; -import com.uber.hoodie.common.table.timeline.HoodieArchivedTimeline; -import com.uber.hoodie.common.table.timeline.HoodieInstant; -import com.uber.hoodie.common.util.AvroUtils; -import com.uber.hoodie.common.util.Option; -import com.uber.hoodie.config.HoodieWriteConfig; -import com.uber.hoodie.exception.HoodieCommitException; -import com.uber.hoodie.exception.HoodieException; -import com.uber.hoodie.exception.HoodieIOException; -import com.uber.hoodie.table.HoodieTable; import java.io.IOException; import java.util.ArrayList; import java.util.Comparator; @@ -59,6 +32,31 @@ import java.util.stream.Stream; import org.apache.avro.Schema; import org.apache.avro.generic.IndexedRecord; import org.apache.hadoop.fs.Path; +import org.apache.hudi.avro.model.HoodieArchivedMetaEntry; +import org.apache.hudi.avro.model.HoodieCleanMetadata; +import org.apache.hudi.avro.model.HoodieRollbackMetadata; +import org.apache.hudi.avro.model.HoodieSavepointMetadata; +import org.apache.hudi.common.model.ActionType; +import org.apache.hudi.common.model.HoodieArchivedLogFile; +import org.apache.hudi.common.model.HoodieCommitMetadata; +import org.apache.hudi.common.model.HoodieRollingStatMetadata; +import org.apache.hudi.common.table.HoodieTableMetaClient; +import org.apache.hudi.common.table.HoodieTimeline; +import org.apache.hudi.common.table.log.HoodieLogFormat; +import org.apache.hudi.common.table.log.HoodieLogFormat.Writer; +import org.apache.hudi.common.table.log.block.HoodieAvroDataBlock; +import org.apache.hudi.common.table.log.block.HoodieLogBlock; +import org.apache.hudi.common.table.log.block.HoodieLogBlock.HeaderMetadataType; +import org.apache.hudi.common.table.timeline.HoodieActiveTimeline; +import org.apache.hudi.common.table.timeline.HoodieArchivedTimeline; +import org.apache.hudi.common.table.timeline.HoodieInstant; +import org.apache.hudi.common.util.AvroUtils; +import org.apache.hudi.common.util.Option; +import org.apache.hudi.config.HoodieWriteConfig; +import org.apache.hudi.exception.HoodieCommitException; +import org.apache.hudi.exception.HoodieException; +import org.apache.hudi.exception.HoodieIOException; +import org.apache.hudi.table.HoodieTable; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; import org.apache.spark.api.java.JavaSparkContext; @@ -73,7 +71,7 @@ public class HoodieCommitArchiveLog { private final Path archiveFilePath; private final HoodieTableMetaClient metaClient; private final HoodieWriteConfig config; - private HoodieLogFormat.Writer writer; + private Writer writer; public HoodieCommitArchiveLog(HoodieWriteConfig config, HoodieTableMetaClient metaClient) { this.config = config; @@ -81,7 +79,7 @@ public class HoodieCommitArchiveLog { this.archiveFilePath = HoodieArchivedTimeline.getArchiveLogPath(metaClient.getArchivePath()); } - private HoodieLogFormat.Writer openWriter() { + private Writer openWriter() { try { if (this.writer == null) { return HoodieLogFormat.newWriterBuilder().onParentPath(archiveFilePath.getParent()) @@ -200,7 +198,8 @@ public class HoodieCommitArchiveLog { Option.fromJavaOptional(archivedInstants.stream() .filter(i -> { return i.isCompleted() - && (i.getAction().equals(COMMIT_ACTION) || (i.getAction().equals(DELTA_COMMIT_ACTION))); + && (i.getAction().equals(HoodieTimeline.COMMIT_ACTION) || (i.getAction().equals( + HoodieTimeline.DELTA_COMMIT_ACTION))); }).max(Comparator.comparing(HoodieInstant::getTimestamp))); if (latestCommitted.isPresent()) { success &= deleteAllInstantsOlderorEqualsInAuxMetaFolder(latestCommitted.get()); @@ -224,7 +223,7 @@ public class HoodieCommitArchiveLog { List instantsToBeDeleted = instants.stream().filter(instant1 -> HoodieTimeline.compareTimestamps(instant1.getTimestamp(), - thresholdInstant.getTimestamp(), LESSER_OR_EQUAL)).collect(Collectors.toList()); + thresholdInstant.getTimestamp(), HoodieTimeline.LESSER_OR_EQUAL)).collect(Collectors.toList()); boolean success = true; for (HoodieInstant deleteInstant : instantsToBeDeleted) { @@ -270,7 +269,7 @@ public class HoodieCommitArchiveLog { private void writeToFile(Schema wrapperSchema, List records) throws Exception { if (records.size() > 0) { - Map header = Maps.newHashMap(); + Map header = Maps.newHashMap(); header.put(HoodieLogBlock.HeaderMetadataType.SCHEMA, wrapperSchema.toString()); HoodieAvroDataBlock block = new HoodieAvroDataBlock(records, header); this.writer = writer.appendBlock(block); @@ -290,7 +289,7 @@ public class HoodieCommitArchiveLog { archivedMetaWrapper.setActionType(ActionType.clean.name()); break; } - case COMMIT_ACTION: { + case HoodieTimeline.COMMIT_ACTION: { HoodieCommitMetadata commitMetadata = HoodieCommitMetadata .fromBytes(commitTimeline.getInstantDetails(hoodieInstant).get(), HoodieCommitMetadata.class); archivedMetaWrapper.setHoodieCommitMetadata(commitMetadataConverter(commitMetadata)); @@ -324,13 +323,13 @@ public class HoodieCommitArchiveLog { return archivedMetaWrapper; } - private com.uber.hoodie.avro.model.HoodieCommitMetadata commitMetadataConverter( + private org.apache.hudi.avro.model.HoodieCommitMetadata commitMetadataConverter( HoodieCommitMetadata hoodieCommitMetadata) { ObjectMapper mapper = new ObjectMapper(); //Need this to ignore other public get() methods mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); - com.uber.hoodie.avro.model.HoodieCommitMetadata avroMetaData = mapper - .convertValue(hoodieCommitMetadata, com.uber.hoodie.avro.model.HoodieCommitMetadata.class); + org.apache.hudi.avro.model.HoodieCommitMetadata avroMetaData = mapper + .convertValue(hoodieCommitMetadata, org.apache.hudi.avro.model.HoodieCommitMetadata.class); // Do not archive Rolling Stats, cannot set to null since AVRO will throw null pointer avroMetaData.getExtraMetadata().put(HoodieRollingStatMetadata.ROLLING_STAT_METADATA_KEY, ""); return avroMetaData; diff --git a/hoodie-client/src/main/java/com/uber/hoodie/io/HoodieCreateHandle.java b/hudi-client/src/main/java/org/apache/hudi/io/HoodieCreateHandle.java similarity index 89% rename from hoodie-client/src/main/java/com/uber/hoodie/io/HoodieCreateHandle.java rename to hudi-client/src/main/java/org/apache/hudi/io/HoodieCreateHandle.java index 317a8faea..4cb935ff7 100644 --- a/hoodie-client/src/main/java/com/uber/hoodie/io/HoodieCreateHandle.java +++ b/hudi-client/src/main/java/org/apache/hudi/io/HoodieCreateHandle.java @@ -16,27 +16,27 @@ * limitations under the License. */ -package com.uber.hoodie.io; +package org.apache.hudi.io; -import com.uber.hoodie.WriteStatus; -import com.uber.hoodie.common.model.HoodiePartitionMetadata; -import com.uber.hoodie.common.model.HoodieRecord; -import com.uber.hoodie.common.model.HoodieRecordLocation; -import com.uber.hoodie.common.model.HoodieRecordPayload; -import com.uber.hoodie.common.model.HoodieWriteStat; -import com.uber.hoodie.common.model.HoodieWriteStat.RuntimeStats; -import com.uber.hoodie.common.util.FSUtils; -import com.uber.hoodie.common.util.Option; -import com.uber.hoodie.config.HoodieWriteConfig; -import com.uber.hoodie.exception.HoodieInsertException; -import com.uber.hoodie.io.storage.HoodieStorageWriter; -import com.uber.hoodie.io.storage.HoodieStorageWriterFactory; -import com.uber.hoodie.table.HoodieTable; import java.io.IOException; import java.util.Iterator; import org.apache.avro.generic.GenericRecord; import org.apache.avro.generic.IndexedRecord; import org.apache.hadoop.fs.Path; +import org.apache.hudi.WriteStatus; +import org.apache.hudi.common.model.HoodiePartitionMetadata; +import org.apache.hudi.common.model.HoodieRecord; +import org.apache.hudi.common.model.HoodieRecordLocation; +import org.apache.hudi.common.model.HoodieRecordPayload; +import org.apache.hudi.common.model.HoodieWriteStat; +import org.apache.hudi.common.model.HoodieWriteStat.RuntimeStats; +import org.apache.hudi.common.util.FSUtils; +import org.apache.hudi.common.util.Option; +import org.apache.hudi.config.HoodieWriteConfig; +import org.apache.hudi.exception.HoodieInsertException; +import org.apache.hudi.io.storage.HoodieStorageWriter; +import org.apache.hudi.io.storage.HoodieStorageWriterFactory; +import org.apache.hudi.table.HoodieTable; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; import org.apache.spark.TaskContext; diff --git a/hoodie-client/src/main/java/com/uber/hoodie/io/HoodieIOHandle.java b/hudi-client/src/main/java/org/apache/hudi/io/HoodieIOHandle.java similarity index 88% rename from hoodie-client/src/main/java/com/uber/hoodie/io/HoodieIOHandle.java rename to hudi-client/src/main/java/org/apache/hudi/io/HoodieIOHandle.java index 89d0aa259..47491b3c9 100644 --- a/hoodie-client/src/main/java/com/uber/hoodie/io/HoodieIOHandle.java +++ b/hudi-client/src/main/java/org/apache/hudi/io/HoodieIOHandle.java @@ -16,12 +16,12 @@ * limitations under the License. */ -package com.uber.hoodie.io; +package org.apache.hudi.io; -import com.uber.hoodie.common.model.HoodieRecordPayload; -import com.uber.hoodie.config.HoodieWriteConfig; -import com.uber.hoodie.table.HoodieTable; import org.apache.hadoop.fs.FileSystem; +import org.apache.hudi.common.model.HoodieRecordPayload; +import org.apache.hudi.config.HoodieWriteConfig; +import org.apache.hudi.table.HoodieTable; public abstract class HoodieIOHandle { diff --git a/hoodie-client/src/main/java/com/uber/hoodie/io/HoodieKeyLookupHandle.java b/hudi-client/src/main/java/org/apache/hudi/io/HoodieKeyLookupHandle.java similarity index 91% rename from hoodie-client/src/main/java/com/uber/hoodie/io/HoodieKeyLookupHandle.java rename to hudi-client/src/main/java/org/apache/hudi/io/HoodieKeyLookupHandle.java index c5bdbf6e8..dec488449 100644 --- a/hoodie-client/src/main/java/com/uber/hoodie/io/HoodieKeyLookupHandle.java +++ b/hudi-client/src/main/java/org/apache/hudi/io/HoodieKeyLookupHandle.java @@ -16,24 +16,24 @@ * limitations under the License. */ -package com.uber.hoodie.io; +package org.apache.hudi.io; -import com.uber.hoodie.common.BloomFilter; -import com.uber.hoodie.common.model.HoodieDataFile; -import com.uber.hoodie.common.model.HoodieRecordPayload; -import com.uber.hoodie.common.model.HoodieTableType; -import com.uber.hoodie.common.util.HoodieTimer; -import com.uber.hoodie.common.util.ParquetUtils; -import com.uber.hoodie.common.util.collection.Pair; -import com.uber.hoodie.config.HoodieWriteConfig; -import com.uber.hoodie.exception.HoodieIndexException; -import com.uber.hoodie.table.HoodieTable; import java.util.ArrayList; import java.util.HashSet; import java.util.List; import java.util.Set; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; +import org.apache.hudi.common.BloomFilter; +import org.apache.hudi.common.model.HoodieDataFile; +import org.apache.hudi.common.model.HoodieRecordPayload; +import org.apache.hudi.common.model.HoodieTableType; +import org.apache.hudi.common.util.HoodieTimer; +import org.apache.hudi.common.util.ParquetUtils; +import org.apache.hudi.common.util.collection.Pair; +import org.apache.hudi.config.HoodieWriteConfig; +import org.apache.hudi.exception.HoodieIndexException; +import org.apache.hudi.table.HoodieTable; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; diff --git a/hoodie-client/src/main/java/com/uber/hoodie/io/HoodieMergeHandle.java b/hudi-client/src/main/java/org/apache/hudi/io/HoodieMergeHandle.java similarity index 92% rename from hoodie-client/src/main/java/com/uber/hoodie/io/HoodieMergeHandle.java rename to hudi-client/src/main/java/org/apache/hudi/io/HoodieMergeHandle.java index 22c573713..95335da6c 100644 --- a/hoodie-client/src/main/java/com/uber/hoodie/io/HoodieMergeHandle.java +++ b/hudi-client/src/main/java/org/apache/hudi/io/HoodieMergeHandle.java @@ -16,28 +16,8 @@ * limitations under the License. */ -package com.uber.hoodie.io; +package org.apache.hudi.io; -import com.uber.hoodie.WriteStatus; -import com.uber.hoodie.common.model.HoodieDataFile; -import com.uber.hoodie.common.model.HoodiePartitionMetadata; -import com.uber.hoodie.common.model.HoodieRecord; -import com.uber.hoodie.common.model.HoodieRecordLocation; -import com.uber.hoodie.common.model.HoodieRecordPayload; -import com.uber.hoodie.common.model.HoodieWriteStat; -import com.uber.hoodie.common.model.HoodieWriteStat.RuntimeStats; -import com.uber.hoodie.common.util.DefaultSizeEstimator; -import com.uber.hoodie.common.util.FSUtils; -import com.uber.hoodie.common.util.HoodieAvroUtils; -import com.uber.hoodie.common.util.HoodieRecordSizeEstimator; -import com.uber.hoodie.common.util.Option; -import com.uber.hoodie.common.util.collection.ExternalSpillableMap; -import com.uber.hoodie.config.HoodieWriteConfig; -import com.uber.hoodie.exception.HoodieIOException; -import com.uber.hoodie.exception.HoodieUpsertException; -import com.uber.hoodie.io.storage.HoodieStorageWriter; -import com.uber.hoodie.io.storage.HoodieStorageWriterFactory; -import com.uber.hoodie.table.HoodieTable; import java.io.IOException; import java.util.HashSet; import java.util.Iterator; @@ -47,6 +27,26 @@ import org.apache.avro.Schema; import org.apache.avro.generic.GenericRecord; import org.apache.avro.generic.IndexedRecord; import org.apache.hadoop.fs.Path; +import org.apache.hudi.WriteStatus; +import org.apache.hudi.common.model.HoodieDataFile; +import org.apache.hudi.common.model.HoodiePartitionMetadata; +import org.apache.hudi.common.model.HoodieRecord; +import org.apache.hudi.common.model.HoodieRecordLocation; +import org.apache.hudi.common.model.HoodieRecordPayload; +import org.apache.hudi.common.model.HoodieWriteStat; +import org.apache.hudi.common.model.HoodieWriteStat.RuntimeStats; +import org.apache.hudi.common.util.DefaultSizeEstimator; +import org.apache.hudi.common.util.FSUtils; +import org.apache.hudi.common.util.HoodieAvroUtils; +import org.apache.hudi.common.util.HoodieRecordSizeEstimator; +import org.apache.hudi.common.util.Option; +import org.apache.hudi.common.util.collection.ExternalSpillableMap; +import org.apache.hudi.config.HoodieWriteConfig; +import org.apache.hudi.exception.HoodieIOException; +import org.apache.hudi.exception.HoodieUpsertException; +import org.apache.hudi.io.storage.HoodieStorageWriter; +import org.apache.hudi.io.storage.HoodieStorageWriterFactory; +import org.apache.hudi.table.HoodieTable; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; import org.apache.spark.TaskContext; diff --git a/hoodie-client/src/main/java/com/uber/hoodie/io/HoodieRangeInfoHandle.java b/hudi-client/src/main/java/org/apache/hudi/io/HoodieRangeInfoHandle.java similarity index 80% rename from hoodie-client/src/main/java/com/uber/hoodie/io/HoodieRangeInfoHandle.java rename to hudi-client/src/main/java/org/apache/hudi/io/HoodieRangeInfoHandle.java index 1f2369e6b..8ae9f5b78 100644 --- a/hoodie-client/src/main/java/com/uber/hoodie/io/HoodieRangeInfoHandle.java +++ b/hudi-client/src/main/java/org/apache/hudi/io/HoodieRangeInfoHandle.java @@ -16,15 +16,15 @@ * limitations under the License. */ -package com.uber.hoodie.io; +package org.apache.hudi.io; -import com.uber.hoodie.common.model.HoodieDataFile; -import com.uber.hoodie.common.model.HoodieRecordPayload; -import com.uber.hoodie.common.util.ParquetUtils; -import com.uber.hoodie.common.util.collection.Pair; -import com.uber.hoodie.config.HoodieWriteConfig; -import com.uber.hoodie.table.HoodieTable; import org.apache.hadoop.fs.Path; +import org.apache.hudi.common.model.HoodieDataFile; +import org.apache.hudi.common.model.HoodieRecordPayload; +import org.apache.hudi.common.util.ParquetUtils; +import org.apache.hudi.common.util.collection.Pair; +import org.apache.hudi.config.HoodieWriteConfig; +import org.apache.hudi.table.HoodieTable; /** * Extract range information for a given file slice diff --git a/hoodie-client/src/main/java/com/uber/hoodie/io/HoodieReadHandle.java b/hudi-client/src/main/java/org/apache/hudi/io/HoodieReadHandle.java similarity index 86% rename from hoodie-client/src/main/java/com/uber/hoodie/io/HoodieReadHandle.java rename to hudi-client/src/main/java/org/apache/hudi/io/HoodieReadHandle.java index e4b18383e..fc91e80bf 100644 --- a/hoodie-client/src/main/java/com/uber/hoodie/io/HoodieReadHandle.java +++ b/hudi-client/src/main/java/org/apache/hudi/io/HoodieReadHandle.java @@ -16,14 +16,14 @@ * limitations under the License. */ -package com.uber.hoodie.io; +package org.apache.hudi.io; -import com.uber.hoodie.common.model.HoodieDataFile; -import com.uber.hoodie.common.model.HoodieRecordPayload; -import com.uber.hoodie.common.util.collection.Pair; -import com.uber.hoodie.config.HoodieWriteConfig; -import com.uber.hoodie.table.HoodieTable; import org.apache.hadoop.fs.FileSystem; +import org.apache.hudi.common.model.HoodieDataFile; +import org.apache.hudi.common.model.HoodieRecordPayload; +import org.apache.hudi.common.util.collection.Pair; +import org.apache.hudi.config.HoodieWriteConfig; +import org.apache.hudi.table.HoodieTable; /** * Base class for read operations done logically on the file group. diff --git a/hoodie-client/src/main/java/com/uber/hoodie/io/HoodieWriteHandle.java b/hudi-client/src/main/java/org/apache/hudi/io/HoodieWriteHandle.java similarity index 90% rename from hoodie-client/src/main/java/com/uber/hoodie/io/HoodieWriteHandle.java rename to hudi-client/src/main/java/org/apache/hudi/io/HoodieWriteHandle.java index 9ec09598d..4cfd941f0 100644 --- a/hoodie-client/src/main/java/com/uber/hoodie/io/HoodieWriteHandle.java +++ b/hudi-client/src/main/java/org/apache/hudi/io/HoodieWriteHandle.java @@ -16,26 +16,26 @@ * limitations under the License. */ -package com.uber.hoodie.io; +package org.apache.hudi.io; -import com.uber.hoodie.WriteStatus; -import com.uber.hoodie.common.model.HoodieRecord; -import com.uber.hoodie.common.model.HoodieRecordPayload; -import com.uber.hoodie.common.util.FSUtils; -import com.uber.hoodie.common.util.HoodieAvroUtils; -import com.uber.hoodie.common.util.HoodieTimer; -import com.uber.hoodie.common.util.Option; -import com.uber.hoodie.common.util.ReflectionUtils; -import com.uber.hoodie.config.HoodieWriteConfig; -import com.uber.hoodie.exception.HoodieException; -import com.uber.hoodie.exception.HoodieIOException; -import com.uber.hoodie.table.HoodieTable; import java.io.IOException; import org.apache.avro.Schema; import org.apache.avro.generic.GenericRecord; import org.apache.avro.generic.IndexedRecord; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hudi.WriteStatus; +import org.apache.hudi.common.model.HoodieRecord; +import org.apache.hudi.common.model.HoodieRecordPayload; +import org.apache.hudi.common.util.FSUtils; +import org.apache.hudi.common.util.HoodieAvroUtils; +import org.apache.hudi.common.util.HoodieTimer; +import org.apache.hudi.common.util.Option; +import org.apache.hudi.common.util.ReflectionUtils; +import org.apache.hudi.config.HoodieWriteConfig; +import org.apache.hudi.exception.HoodieException; +import org.apache.hudi.exception.HoodieIOException; +import org.apache.hudi.table.HoodieTable; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; import org.apache.spark.TaskContext; diff --git a/hoodie-client/src/main/java/com/uber/hoodie/io/compact/HoodieCompactor.java b/hudi-client/src/main/java/org/apache/hudi/io/compact/HoodieCompactor.java similarity index 88% rename from hoodie-client/src/main/java/com/uber/hoodie/io/compact/HoodieCompactor.java rename to hudi-client/src/main/java/org/apache/hudi/io/compact/HoodieCompactor.java index f08e8b65c..f57758f89 100644 --- a/hoodie-client/src/main/java/com/uber/hoodie/io/compact/HoodieCompactor.java +++ b/hudi-client/src/main/java/org/apache/hudi/io/compact/HoodieCompactor.java @@ -16,16 +16,16 @@ * limitations under the License. */ -package com.uber.hoodie.io.compact; +package org.apache.hudi.io.compact; -import com.uber.hoodie.WriteStatus; -import com.uber.hoodie.avro.model.HoodieCompactionPlan; -import com.uber.hoodie.common.model.HoodieFileGroupId; -import com.uber.hoodie.config.HoodieWriteConfig; -import com.uber.hoodie.table.HoodieTable; import java.io.IOException; import java.io.Serializable; import java.util.Set; +import org.apache.hudi.WriteStatus; +import org.apache.hudi.avro.model.HoodieCompactionPlan; +import org.apache.hudi.common.model.HoodieFileGroupId; +import org.apache.hudi.config.HoodieWriteConfig; +import org.apache.hudi.table.HoodieTable; import org.apache.spark.api.java.JavaRDD; import org.apache.spark.api.java.JavaSparkContext; diff --git a/hoodie-client/src/main/java/com/uber/hoodie/io/compact/HoodieRealtimeTableCompactor.java b/hudi-client/src/main/java/org/apache/hudi/io/compact/HoodieRealtimeTableCompactor.java similarity index 89% rename from hoodie-client/src/main/java/com/uber/hoodie/io/compact/HoodieRealtimeTableCompactor.java rename to hudi-client/src/main/java/org/apache/hudi/io/compact/HoodieRealtimeTableCompactor.java index 923f436d6..7963857d9 100644 --- a/hoodie-client/src/main/java/com/uber/hoodie/io/compact/HoodieRealtimeTableCompactor.java +++ b/hudi-client/src/main/java/org/apache/hudi/io/compact/HoodieRealtimeTableCompactor.java @@ -16,35 +16,13 @@ * limitations under the License. */ -package com.uber.hoodie.io.compact; +package org.apache.hudi.io.compact; import static java.util.stream.Collectors.toList; import com.google.common.base.Preconditions; import com.google.common.collect.Lists; import com.google.common.collect.Sets; -import com.uber.hoodie.WriteStatus; -import com.uber.hoodie.avro.model.HoodieCompactionOperation; -import com.uber.hoodie.avro.model.HoodieCompactionPlan; -import com.uber.hoodie.common.model.CompactionOperation; -import com.uber.hoodie.common.model.HoodieDataFile; -import com.uber.hoodie.common.model.HoodieFileGroupId; -import com.uber.hoodie.common.model.HoodieLogFile; -import com.uber.hoodie.common.model.HoodieTableType; -import com.uber.hoodie.common.model.HoodieWriteStat.RuntimeStats; -import com.uber.hoodie.common.table.HoodieTableMetaClient; -import com.uber.hoodie.common.table.HoodieTimeline; -import com.uber.hoodie.common.table.TableFileSystemView; -import com.uber.hoodie.common.table.log.HoodieMergedLogRecordScanner; -import com.uber.hoodie.common.util.CompactionUtils; -import com.uber.hoodie.common.util.FSUtils; -import com.uber.hoodie.common.util.HoodieAvroUtils; -import com.uber.hoodie.common.util.Option; -import com.uber.hoodie.common.util.collection.Pair; -import com.uber.hoodie.config.HoodieWriteConfig; -import com.uber.hoodie.io.compact.strategy.CompactionStrategy; -import com.uber.hoodie.table.HoodieCopyOnWriteTable; -import com.uber.hoodie.table.HoodieTable; import java.io.IOException; import java.util.Collection; import java.util.Iterator; @@ -54,6 +32,28 @@ import java.util.stream.Collectors; import java.util.stream.StreamSupport; import org.apache.avro.Schema; import org.apache.hadoop.fs.FileSystem; +import org.apache.hudi.WriteStatus; +import org.apache.hudi.avro.model.HoodieCompactionOperation; +import org.apache.hudi.avro.model.HoodieCompactionPlan; +import org.apache.hudi.common.model.CompactionOperation; +import org.apache.hudi.common.model.HoodieDataFile; +import org.apache.hudi.common.model.HoodieFileGroupId; +import org.apache.hudi.common.model.HoodieLogFile; +import org.apache.hudi.common.model.HoodieTableType; +import org.apache.hudi.common.model.HoodieWriteStat.RuntimeStats; +import org.apache.hudi.common.table.HoodieTableMetaClient; +import org.apache.hudi.common.table.HoodieTimeline; +import org.apache.hudi.common.table.TableFileSystemView.RealtimeView; +import org.apache.hudi.common.table.log.HoodieMergedLogRecordScanner; +import org.apache.hudi.common.util.CompactionUtils; +import org.apache.hudi.common.util.FSUtils; +import org.apache.hudi.common.util.HoodieAvroUtils; +import org.apache.hudi.common.util.Option; +import org.apache.hudi.common.util.collection.Pair; +import org.apache.hudi.config.HoodieWriteConfig; +import org.apache.hudi.io.compact.strategy.CompactionStrategy; +import org.apache.hudi.table.HoodieCopyOnWriteTable; +import org.apache.hudi.table.HoodieTable; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; import org.apache.spark.api.java.JavaRDD; @@ -190,7 +190,7 @@ public class HoodieRealtimeTableCompactor implements HoodieCompactor { return null; } - TableFileSystemView.RealtimeView fileSystemView = hoodieTable.getRTFileSystemView(); + RealtimeView fileSystemView = hoodieTable.getRTFileSystemView(); log.info("Compaction looking for files to compact in " + partitionPaths + " partitions"); List operations = jsc.parallelize(partitionPaths, partitionPaths.size()) diff --git a/hoodie-client/src/main/java/com/uber/hoodie/io/compact/strategy/BoundedIOCompactionStrategy.java b/hudi-client/src/main/java/org/apache/hudi/io/compact/strategy/BoundedIOCompactionStrategy.java similarity index 90% rename from hoodie-client/src/main/java/com/uber/hoodie/io/compact/strategy/BoundedIOCompactionStrategy.java rename to hudi-client/src/main/java/org/apache/hudi/io/compact/strategy/BoundedIOCompactionStrategy.java index 323ce7c86..ad5451005 100644 --- a/hoodie-client/src/main/java/com/uber/hoodie/io/compact/strategy/BoundedIOCompactionStrategy.java +++ b/hudi-client/src/main/java/org/apache/hudi/io/compact/strategy/BoundedIOCompactionStrategy.java @@ -16,13 +16,13 @@ * limitations under the License. */ -package com.uber.hoodie.io.compact.strategy; +package org.apache.hudi.io.compact.strategy; import com.google.common.collect.Lists; -import com.uber.hoodie.avro.model.HoodieCompactionOperation; -import com.uber.hoodie.avro.model.HoodieCompactionPlan; -import com.uber.hoodie.config.HoodieWriteConfig; import java.util.List; +import org.apache.hudi.avro.model.HoodieCompactionOperation; +import org.apache.hudi.avro.model.HoodieCompactionPlan; +import org.apache.hudi.config.HoodieWriteConfig; /** * CompactionStrategy which looks at total IO to be done for the compaction (read + write) and diff --git a/hoodie-client/src/main/java/com/uber/hoodie/io/compact/strategy/BoundedPartitionAwareCompactionStrategy.java b/hudi-client/src/main/java/org/apache/hudi/io/compact/strategy/BoundedPartitionAwareCompactionStrategy.java similarity index 94% rename from hoodie-client/src/main/java/com/uber/hoodie/io/compact/strategy/BoundedPartitionAwareCompactionStrategy.java rename to hudi-client/src/main/java/org/apache/hudi/io/compact/strategy/BoundedPartitionAwareCompactionStrategy.java index 7bd1c00db..6f0f793a9 100644 --- a/hoodie-client/src/main/java/com/uber/hoodie/io/compact/strategy/BoundedPartitionAwareCompactionStrategy.java +++ b/hudi-client/src/main/java/org/apache/hudi/io/compact/strategy/BoundedPartitionAwareCompactionStrategy.java @@ -16,11 +16,8 @@ * limitations under the License. */ -package com.uber.hoodie.io.compact.strategy; +package org.apache.hudi.io.compact.strategy; -import com.uber.hoodie.avro.model.HoodieCompactionOperation; -import com.uber.hoodie.avro.model.HoodieCompactionPlan; -import com.uber.hoodie.config.HoodieWriteConfig; import java.text.SimpleDateFormat; import java.util.Comparator; import java.util.Date; @@ -28,6 +25,9 @@ import java.util.List; import java.util.Map; import java.util.stream.Collectors; import org.apache.commons.lang3.time.DateUtils; +import org.apache.hudi.avro.model.HoodieCompactionOperation; +import org.apache.hudi.avro.model.HoodieCompactionPlan; +import org.apache.hudi.config.HoodieWriteConfig; /** * This strategy ensures that the last N partitions are picked up even if there are later partitions created for the diff --git a/hoodie-client/src/main/java/com/uber/hoodie/io/compact/strategy/CompactionStrategy.java b/hudi-client/src/main/java/org/apache/hudi/io/compact/strategy/CompactionStrategy.java similarity index 91% rename from hoodie-client/src/main/java/com/uber/hoodie/io/compact/strategy/CompactionStrategy.java rename to hudi-client/src/main/java/org/apache/hudi/io/compact/strategy/CompactionStrategy.java index 9db21022a..b7faf074e 100644 --- a/hoodie-client/src/main/java/com/uber/hoodie/io/compact/strategy/CompactionStrategy.java +++ b/hudi-client/src/main/java/org/apache/hudi/io/compact/strategy/CompactionStrategy.java @@ -16,19 +16,20 @@ * limitations under the License. */ -package com.uber.hoodie.io.compact.strategy; +package org.apache.hudi.io.compact.strategy; import com.google.common.collect.Maps; -import com.uber.hoodie.avro.model.HoodieCompactionOperation; -import com.uber.hoodie.avro.model.HoodieCompactionPlan; -import com.uber.hoodie.common.model.HoodieDataFile; -import com.uber.hoodie.common.model.HoodieLogFile; -import com.uber.hoodie.common.util.FSUtils; -import com.uber.hoodie.common.util.Option; -import com.uber.hoodie.config.HoodieWriteConfig; import java.io.Serializable; import java.util.List; import java.util.Map; +import org.apache.hudi.avro.model.HoodieCompactionOperation; +import org.apache.hudi.avro.model.HoodieCompactionPlan; +import org.apache.hudi.common.model.HoodieDataFile; +import org.apache.hudi.common.model.HoodieLogFile; +import org.apache.hudi.common.util.FSUtils; +import org.apache.hudi.common.util.Option; +import org.apache.hudi.config.HoodieWriteConfig; +import org.apache.hudi.io.compact.HoodieRealtimeTableCompactor; /** * Strategy for compaction. Pluggable implementation to define how compaction should be done. The @@ -37,7 +38,7 @@ import java.util.Map; * Implementation of CompactionStrategy cannot hold any state. Difference instantiations can be * passed in every time * - * @see com.uber.hoodie.io.compact.HoodieRealtimeTableCompactor + * @see HoodieRealtimeTableCompactor */ public abstract class CompactionStrategy implements Serializable { diff --git a/hoodie-client/src/main/java/com/uber/hoodie/io/compact/strategy/DayBasedCompactionStrategy.java b/hudi-client/src/main/java/org/apache/hudi/io/compact/strategy/DayBasedCompactionStrategy.java similarity index 92% rename from hoodie-client/src/main/java/com/uber/hoodie/io/compact/strategy/DayBasedCompactionStrategy.java rename to hudi-client/src/main/java/org/apache/hudi/io/compact/strategy/DayBasedCompactionStrategy.java index 15be0ae9f..b04574bdc 100644 --- a/hoodie-client/src/main/java/com/uber/hoodie/io/compact/strategy/DayBasedCompactionStrategy.java +++ b/hudi-client/src/main/java/org/apache/hudi/io/compact/strategy/DayBasedCompactionStrategy.java @@ -17,13 +17,9 @@ */ -package com.uber.hoodie.io.compact.strategy; +package org.apache.hudi.io.compact.strategy; import com.google.common.annotations.VisibleForTesting; -import com.uber.hoodie.avro.model.HoodieCompactionOperation; -import com.uber.hoodie.avro.model.HoodieCompactionPlan; -import com.uber.hoodie.config.HoodieWriteConfig; -import com.uber.hoodie.exception.HoodieException; import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.Comparator; @@ -32,6 +28,10 @@ import java.util.List; import java.util.Locale; import java.util.Map; import java.util.stream.Collectors; +import org.apache.hudi.avro.model.HoodieCompactionOperation; +import org.apache.hudi.avro.model.HoodieCompactionPlan; +import org.apache.hudi.config.HoodieWriteConfig; +import org.apache.hudi.exception.HoodieException; /** * This strategy orders compactions in reverse order of creation of Hive Partitions. It helps to diff --git a/hoodie-client/src/main/java/com/uber/hoodie/io/compact/strategy/LogFileSizeBasedCompactionStrategy.java b/hudi-client/src/main/java/org/apache/hudi/io/compact/strategy/LogFileSizeBasedCompactionStrategy.java similarity index 88% rename from hoodie-client/src/main/java/com/uber/hoodie/io/compact/strategy/LogFileSizeBasedCompactionStrategy.java rename to hudi-client/src/main/java/org/apache/hudi/io/compact/strategy/LogFileSizeBasedCompactionStrategy.java index e5a4061a0..829112260 100644 --- a/hoodie-client/src/main/java/com/uber/hoodie/io/compact/strategy/LogFileSizeBasedCompactionStrategy.java +++ b/hudi-client/src/main/java/org/apache/hudi/io/compact/strategy/LogFileSizeBasedCompactionStrategy.java @@ -16,18 +16,18 @@ * limitations under the License. */ -package com.uber.hoodie.io.compact.strategy; +package org.apache.hudi.io.compact.strategy; -import com.uber.hoodie.avro.model.HoodieCompactionOperation; -import com.uber.hoodie.avro.model.HoodieCompactionPlan; -import com.uber.hoodie.common.model.HoodieDataFile; -import com.uber.hoodie.common.model.HoodieLogFile; -import com.uber.hoodie.common.util.Option; -import com.uber.hoodie.config.HoodieWriteConfig; import java.util.Comparator; import java.util.List; import java.util.Map; import java.util.stream.Collectors; +import org.apache.hudi.avro.model.HoodieCompactionOperation; +import org.apache.hudi.avro.model.HoodieCompactionPlan; +import org.apache.hudi.common.model.HoodieDataFile; +import org.apache.hudi.common.model.HoodieLogFile; +import org.apache.hudi.common.util.Option; +import org.apache.hudi.config.HoodieWriteConfig; /** * LogFileSizeBasedCompactionStrategy orders the compactions based on the total log files size and diff --git a/hoodie-client/src/main/java/com/uber/hoodie/io/compact/strategy/UnBoundedCompactionStrategy.java b/hudi-client/src/main/java/org/apache/hudi/io/compact/strategy/UnBoundedCompactionStrategy.java similarity index 86% rename from hoodie-client/src/main/java/com/uber/hoodie/io/compact/strategy/UnBoundedCompactionStrategy.java rename to hudi-client/src/main/java/org/apache/hudi/io/compact/strategy/UnBoundedCompactionStrategy.java index 252e7d41c..0edc6a61a 100644 --- a/hoodie-client/src/main/java/com/uber/hoodie/io/compact/strategy/UnBoundedCompactionStrategy.java +++ b/hudi-client/src/main/java/org/apache/hudi/io/compact/strategy/UnBoundedCompactionStrategy.java @@ -16,12 +16,12 @@ * limitations under the License. */ -package com.uber.hoodie.io.compact.strategy; +package org.apache.hudi.io.compact.strategy; -import com.uber.hoodie.avro.model.HoodieCompactionOperation; -import com.uber.hoodie.avro.model.HoodieCompactionPlan; -import com.uber.hoodie.config.HoodieWriteConfig; import java.util.List; +import org.apache.hudi.avro.model.HoodieCompactionOperation; +import org.apache.hudi.avro.model.HoodieCompactionPlan; +import org.apache.hudi.config.HoodieWriteConfig; /** * UnBoundedCompactionStrategy will not change ordering or filter any compaction. It is a diff --git a/hoodie-client/src/main/java/com/uber/hoodie/io/compact/strategy/UnBoundedPartitionAwareCompactionStrategy.java b/hudi-client/src/main/java/org/apache/hudi/io/compact/strategy/UnBoundedPartitionAwareCompactionStrategy.java similarity index 93% rename from hoodie-client/src/main/java/com/uber/hoodie/io/compact/strategy/UnBoundedPartitionAwareCompactionStrategy.java rename to hudi-client/src/main/java/org/apache/hudi/io/compact/strategy/UnBoundedPartitionAwareCompactionStrategy.java index f2efd4d35..220de5d27 100644 --- a/hoodie-client/src/main/java/com/uber/hoodie/io/compact/strategy/UnBoundedPartitionAwareCompactionStrategy.java +++ b/hudi-client/src/main/java/org/apache/hudi/io/compact/strategy/UnBoundedPartitionAwareCompactionStrategy.java @@ -16,15 +16,15 @@ * limitations under the License. */ -package com.uber.hoodie.io.compact.strategy; +package org.apache.hudi.io.compact.strategy; -import com.uber.hoodie.avro.model.HoodieCompactionOperation; -import com.uber.hoodie.avro.model.HoodieCompactionPlan; -import com.uber.hoodie.config.HoodieWriteConfig; import java.util.ArrayList; import java.util.Comparator; import java.util.List; import java.util.stream.Collectors; +import org.apache.hudi.avro.model.HoodieCompactionOperation; +import org.apache.hudi.avro.model.HoodieCompactionPlan; +import org.apache.hudi.config.HoodieWriteConfig; /** * UnBoundedPartitionAwareCompactionStrategy is a custom UnBounded Strategy. diff --git a/hoodie-client/src/main/java/com/uber/hoodie/io/storage/HoodieParquetConfig.java b/hudi-client/src/main/java/org/apache/hudi/io/storage/HoodieParquetConfig.java similarity index 96% rename from hoodie-client/src/main/java/com/uber/hoodie/io/storage/HoodieParquetConfig.java rename to hudi-client/src/main/java/org/apache/hudi/io/storage/HoodieParquetConfig.java index 7aa9b7e07..93a5182e3 100644 --- a/hoodie-client/src/main/java/com/uber/hoodie/io/storage/HoodieParquetConfig.java +++ b/hudi-client/src/main/java/org/apache/hudi/io/storage/HoodieParquetConfig.java @@ -16,10 +16,10 @@ * limitations under the License. */ -package com.uber.hoodie.io.storage; +package org.apache.hudi.io.storage; -import com.uber.hoodie.avro.HoodieAvroWriteSupport; import org.apache.hadoop.conf.Configuration; +import org.apache.hudi.avro.HoodieAvroWriteSupport; import org.apache.parquet.hadoop.metadata.CompressionCodecName; public class HoodieParquetConfig { diff --git a/hoodie-client/src/main/java/com/uber/hoodie/io/storage/HoodieParquetWriter.java b/hudi-client/src/main/java/org/apache/hudi/io/storage/HoodieParquetWriter.java similarity index 92% rename from hoodie-client/src/main/java/com/uber/hoodie/io/storage/HoodieParquetWriter.java rename to hudi-client/src/main/java/org/apache/hudi/io/storage/HoodieParquetWriter.java index 939e37e3a..d1d5d9f43 100644 --- a/hoodie-client/src/main/java/com/uber/hoodie/io/storage/HoodieParquetWriter.java +++ b/hudi-client/src/main/java/org/apache/hudi/io/storage/HoodieParquetWriter.java @@ -16,14 +16,8 @@ * limitations under the License. */ -package com.uber.hoodie.io.storage; +package org.apache.hudi.io.storage; -import com.uber.hoodie.avro.HoodieAvroWriteSupport; -import com.uber.hoodie.common.io.storage.HoodieWrapperFileSystem; -import com.uber.hoodie.common.model.HoodieRecord; -import com.uber.hoodie.common.model.HoodieRecordPayload; -import com.uber.hoodie.common.util.FSUtils; -import com.uber.hoodie.common.util.HoodieAvroUtils; import java.io.IOException; import java.util.concurrent.atomic.AtomicLong; import org.apache.avro.Schema; @@ -31,6 +25,12 @@ import org.apache.avro.generic.GenericRecord; import org.apache.avro.generic.IndexedRecord; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; +import org.apache.hudi.avro.HoodieAvroWriteSupport; +import org.apache.hudi.common.io.storage.HoodieWrapperFileSystem; +import org.apache.hudi.common.model.HoodieRecord; +import org.apache.hudi.common.model.HoodieRecordPayload; +import org.apache.hudi.common.util.FSUtils; +import org.apache.hudi.common.util.HoodieAvroUtils; import org.apache.parquet.hadoop.ParquetFileWriter; import org.apache.parquet.hadoop.ParquetWriter; import org.apache.spark.TaskContext; diff --git a/hoodie-client/src/main/java/com/uber/hoodie/io/storage/HoodieStorageWriter.java b/hudi-client/src/main/java/org/apache/hudi/io/storage/HoodieStorageWriter.java similarity index 93% rename from hoodie-client/src/main/java/com/uber/hoodie/io/storage/HoodieStorageWriter.java rename to hudi-client/src/main/java/org/apache/hudi/io/storage/HoodieStorageWriter.java index 0621b5987..322544150 100644 --- a/hoodie-client/src/main/java/com/uber/hoodie/io/storage/HoodieStorageWriter.java +++ b/hudi-client/src/main/java/org/apache/hudi/io/storage/HoodieStorageWriter.java @@ -16,11 +16,11 @@ * limitations under the License. */ -package com.uber.hoodie.io.storage; +package org.apache.hudi.io.storage; -import com.uber.hoodie.common.model.HoodieRecord; import java.io.IOException; import org.apache.avro.generic.IndexedRecord; +import org.apache.hudi.common.model.HoodieRecord; public interface HoodieStorageWriter { diff --git a/hoodie-client/src/main/java/com/uber/hoodie/io/storage/HoodieStorageWriterFactory.java b/hudi-client/src/main/java/org/apache/hudi/io/storage/HoodieStorageWriterFactory.java similarity index 89% rename from hoodie-client/src/main/java/com/uber/hoodie/io/storage/HoodieStorageWriterFactory.java rename to hudi-client/src/main/java/org/apache/hudi/io/storage/HoodieStorageWriterFactory.java index 610cda186..a0f8c73a6 100644 --- a/hoodie-client/src/main/java/com/uber/hoodie/io/storage/HoodieStorageWriterFactory.java +++ b/hudi-client/src/main/java/org/apache/hudi/io/storage/HoodieStorageWriterFactory.java @@ -16,17 +16,17 @@ * limitations under the License. */ -package com.uber.hoodie.io.storage; +package org.apache.hudi.io.storage; -import com.uber.hoodie.avro.HoodieAvroWriteSupport; -import com.uber.hoodie.common.BloomFilter; -import com.uber.hoodie.common.model.HoodieRecordPayload; -import com.uber.hoodie.config.HoodieWriteConfig; -import com.uber.hoodie.table.HoodieTable; import java.io.IOException; import org.apache.avro.Schema; import org.apache.avro.generic.IndexedRecord; import org.apache.hadoop.fs.Path; +import org.apache.hudi.avro.HoodieAvroWriteSupport; +import org.apache.hudi.common.BloomFilter; +import org.apache.hudi.common.model.HoodieRecordPayload; +import org.apache.hudi.config.HoodieWriteConfig; +import org.apache.hudi.table.HoodieTable; import org.apache.parquet.avro.AvroSchemaConverter; public class HoodieStorageWriterFactory { diff --git a/hoodie-client/src/main/java/com/uber/hoodie/metrics/HoodieMetrics.java b/hudi-client/src/main/java/org/apache/hudi/metrics/HoodieMetrics.java similarity index 73% rename from hoodie-client/src/main/java/com/uber/hoodie/metrics/HoodieMetrics.java rename to hudi-client/src/main/java/org/apache/hudi/metrics/HoodieMetrics.java index ecc347f4d..a5646349c 100644 --- a/hoodie-client/src/main/java/com/uber/hoodie/metrics/HoodieMetrics.java +++ b/hudi-client/src/main/java/org/apache/hudi/metrics/HoodieMetrics.java @@ -16,15 +16,13 @@ * limitations under the License. */ -package com.uber.hoodie.metrics; - -import static com.uber.hoodie.metrics.Metrics.registerGauge; +package org.apache.hudi.metrics; import com.codahale.metrics.Timer; import com.google.common.annotations.VisibleForTesting; -import com.uber.hoodie.common.model.HoodieCommitMetadata; -import com.uber.hoodie.common.table.HoodieTimeline; -import com.uber.hoodie.config.HoodieWriteConfig; +import org.apache.hudi.common.model.HoodieCommitMetadata; +import org.apache.hudi.common.table.HoodieTimeline; +import org.apache.hudi.config.HoodieWriteConfig; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; @@ -126,21 +124,21 @@ public class HoodieMetrics { long totalCompactedRecordsUpdated = metadata.getTotalCompactedRecordsUpdated(); long totalLogFilesCompacted = metadata.getTotalLogFilesCompacted(); long totalLogFilesSize = metadata.getTotalLogFilesSize(); - registerGauge(getMetricsName(actionType, "duration"), durationInMs); - registerGauge(getMetricsName(actionType, "totalPartitionsWritten"), totalPartitionsWritten); - registerGauge(getMetricsName(actionType, "totalFilesInsert"), totalFilesInsert); - registerGauge(getMetricsName(actionType, "totalFilesUpdate"), totalFilesUpdate); - registerGauge(getMetricsName(actionType, "totalRecordsWritten"), totalRecordsWritten); - registerGauge(getMetricsName(actionType, "totalUpdateRecordsWritten"), totalUpdateRecordsWritten); - registerGauge(getMetricsName(actionType, "totalInsertRecordsWritten"), totalInsertRecordsWritten); - registerGauge(getMetricsName(actionType, "totalBytesWritten"), totalBytesWritten); - registerGauge(getMetricsName(actionType, "commitTime"), commitEpochTimeInMs); - registerGauge(getMetricsName(actionType, "totalScanTime"), totalTimeTakenByScanner); - registerGauge(getMetricsName(actionType, "totalCreateTime"), totalTimeTakenForInsert); - registerGauge(getMetricsName(actionType, "totalUpsertTime"), totalTimeTakenForUpsert); - registerGauge(getMetricsName(actionType, "totalCompactedRecordsUpdated"), totalCompactedRecordsUpdated); - registerGauge(getMetricsName(actionType, "totalLogFilesCompacted"), totalLogFilesCompacted); - registerGauge(getMetricsName(actionType, "totalLogFilesSize"), totalLogFilesSize); + Metrics.registerGauge(getMetricsName(actionType, "duration"), durationInMs); + Metrics.registerGauge(getMetricsName(actionType, "totalPartitionsWritten"), totalPartitionsWritten); + Metrics.registerGauge(getMetricsName(actionType, "totalFilesInsert"), totalFilesInsert); + Metrics.registerGauge(getMetricsName(actionType, "totalFilesUpdate"), totalFilesUpdate); + Metrics.registerGauge(getMetricsName(actionType, "totalRecordsWritten"), totalRecordsWritten); + Metrics.registerGauge(getMetricsName(actionType, "totalUpdateRecordsWritten"), totalUpdateRecordsWritten); + Metrics.registerGauge(getMetricsName(actionType, "totalInsertRecordsWritten"), totalInsertRecordsWritten); + Metrics.registerGauge(getMetricsName(actionType, "totalBytesWritten"), totalBytesWritten); + Metrics.registerGauge(getMetricsName(actionType, "commitTime"), commitEpochTimeInMs); + Metrics.registerGauge(getMetricsName(actionType, "totalScanTime"), totalTimeTakenByScanner); + Metrics.registerGauge(getMetricsName(actionType, "totalCreateTime"), totalTimeTakenForInsert); + Metrics.registerGauge(getMetricsName(actionType, "totalUpsertTime"), totalTimeTakenForUpsert); + Metrics.registerGauge(getMetricsName(actionType, "totalCompactedRecordsUpdated"), totalCompactedRecordsUpdated); + Metrics.registerGauge(getMetricsName(actionType, "totalLogFilesCompacted"), totalLogFilesCompacted); + Metrics.registerGauge(getMetricsName(actionType, "totalLogFilesSize"), totalLogFilesSize); } } @@ -149,8 +147,8 @@ public class HoodieMetrics { logger.info(String .format("Sending rollback metrics (duration=%d, numFilesDeleted=%d)", durationInMs, numFilesDeleted)); - registerGauge(getMetricsName("rollback", "duration"), durationInMs); - registerGauge(getMetricsName("rollback", "numFilesDeleted"), numFilesDeleted); + Metrics.registerGauge(getMetricsName("rollback", "duration"), durationInMs); + Metrics.registerGauge(getMetricsName("rollback", "numFilesDeleted"), numFilesDeleted); } } @@ -159,8 +157,8 @@ public class HoodieMetrics { logger.info(String .format("Sending clean metrics (duration=%d, numFilesDeleted=%d)", durationInMs, numFilesDeleted)); - registerGauge(getMetricsName("clean", "duration"), durationInMs); - registerGauge(getMetricsName("clean", "numFilesDeleted"), numFilesDeleted); + Metrics.registerGauge(getMetricsName("clean", "duration"), durationInMs); + Metrics.registerGauge(getMetricsName("clean", "numFilesDeleted"), numFilesDeleted); } } @@ -169,8 +167,8 @@ public class HoodieMetrics { logger.info(String .format("Sending finalize write metrics (duration=%d, numFilesFinalized=%d)", durationInMs, numFilesFinalized)); - registerGauge(getMetricsName("finalize", "duration"), durationInMs); - registerGauge(getMetricsName("finalize", "numFilesFinalized"), numFilesFinalized); + Metrics.registerGauge(getMetricsName("finalize", "duration"), durationInMs); + Metrics.registerGauge(getMetricsName("finalize", "numFilesFinalized"), numFilesFinalized); } } diff --git a/hoodie-client/src/main/java/com/uber/hoodie/metrics/InMemoryMetricsReporter.java b/hudi-client/src/main/java/org/apache/hudi/metrics/InMemoryMetricsReporter.java similarity index 97% rename from hoodie-client/src/main/java/com/uber/hoodie/metrics/InMemoryMetricsReporter.java rename to hudi-client/src/main/java/org/apache/hudi/metrics/InMemoryMetricsReporter.java index c2235b4a0..91de8dfe3 100644 --- a/hoodie-client/src/main/java/com/uber/hoodie/metrics/InMemoryMetricsReporter.java +++ b/hudi-client/src/main/java/org/apache/hudi/metrics/InMemoryMetricsReporter.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package com.uber.hoodie.metrics; +package org.apache.hudi.metrics; import java.io.Closeable; diff --git a/hoodie-client/src/main/java/com/uber/hoodie/metrics/Metrics.java b/hudi-client/src/main/java/org/apache/hudi/metrics/Metrics.java similarity index 95% rename from hoodie-client/src/main/java/com/uber/hoodie/metrics/Metrics.java rename to hudi-client/src/main/java/org/apache/hudi/metrics/Metrics.java index 15f42081b..82219bb19 100644 --- a/hoodie-client/src/main/java/com/uber/hoodie/metrics/Metrics.java +++ b/hudi-client/src/main/java/org/apache/hudi/metrics/Metrics.java @@ -16,15 +16,15 @@ * limitations under the License. */ -package com.uber.hoodie.metrics; +package org.apache.hudi.metrics; import com.codahale.metrics.Gauge; import com.codahale.metrics.MetricRegistry; import com.google.common.io.Closeables; -import com.uber.hoodie.config.HoodieWriteConfig; -import com.uber.hoodie.exception.HoodieException; import java.io.Closeable; import org.apache.commons.configuration.ConfigurationException; +import org.apache.hudi.config.HoodieWriteConfig; +import org.apache.hudi.exception.HoodieException; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; diff --git a/hoodie-client/src/main/java/com/uber/hoodie/metrics/MetricsGraphiteReporter.java b/hudi-client/src/main/java/org/apache/hudi/metrics/MetricsGraphiteReporter.java similarity index 97% rename from hoodie-client/src/main/java/com/uber/hoodie/metrics/MetricsGraphiteReporter.java rename to hudi-client/src/main/java/org/apache/hudi/metrics/MetricsGraphiteReporter.java index 9fe3f574e..096545c65 100644 --- a/hoodie-client/src/main/java/com/uber/hoodie/metrics/MetricsGraphiteReporter.java +++ b/hudi-client/src/main/java/org/apache/hudi/metrics/MetricsGraphiteReporter.java @@ -16,16 +16,16 @@ * limitations under the License. */ -package com.uber.hoodie.metrics; +package org.apache.hudi.metrics; import com.codahale.metrics.MetricFilter; import com.codahale.metrics.MetricRegistry; import com.codahale.metrics.graphite.Graphite; import com.codahale.metrics.graphite.GraphiteReporter; -import com.uber.hoodie.config.HoodieWriteConfig; import java.io.Closeable; import java.net.InetSocketAddress; import java.util.concurrent.TimeUnit; +import org.apache.hudi.config.HoodieWriteConfig; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; diff --git a/hoodie-client/src/main/java/com/uber/hoodie/metrics/MetricsReporter.java b/hudi-client/src/main/java/org/apache/hudi/metrics/MetricsReporter.java similarity index 97% rename from hoodie-client/src/main/java/com/uber/hoodie/metrics/MetricsReporter.java rename to hudi-client/src/main/java/org/apache/hudi/metrics/MetricsReporter.java index c9b9a5b22..45ce852fa 100644 --- a/hoodie-client/src/main/java/com/uber/hoodie/metrics/MetricsReporter.java +++ b/hudi-client/src/main/java/org/apache/hudi/metrics/MetricsReporter.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package com.uber.hoodie.metrics; +package org.apache.hudi.metrics; import java.io.Closeable; diff --git a/hoodie-client/src/main/java/com/uber/hoodie/metrics/MetricsReporterFactory.java b/hudi-client/src/main/java/org/apache/hudi/metrics/MetricsReporterFactory.java similarity index 95% rename from hoodie-client/src/main/java/com/uber/hoodie/metrics/MetricsReporterFactory.java rename to hudi-client/src/main/java/org/apache/hudi/metrics/MetricsReporterFactory.java index 3afd84ed6..10aa33f45 100644 --- a/hoodie-client/src/main/java/com/uber/hoodie/metrics/MetricsReporterFactory.java +++ b/hudi-client/src/main/java/org/apache/hudi/metrics/MetricsReporterFactory.java @@ -16,10 +16,10 @@ * limitations under the License. */ -package com.uber.hoodie.metrics; +package org.apache.hudi.metrics; import com.codahale.metrics.MetricRegistry; -import com.uber.hoodie.config.HoodieWriteConfig; +import org.apache.hudi.config.HoodieWriteConfig; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; diff --git a/hoodie-client/src/main/java/com/uber/hoodie/metrics/MetricsReporterType.java b/hudi-client/src/main/java/org/apache/hudi/metrics/MetricsReporterType.java similarity index 96% rename from hoodie-client/src/main/java/com/uber/hoodie/metrics/MetricsReporterType.java rename to hudi-client/src/main/java/org/apache/hudi/metrics/MetricsReporterType.java index 32299d9e6..83b64bb23 100644 --- a/hoodie-client/src/main/java/com/uber/hoodie/metrics/MetricsReporterType.java +++ b/hudi-client/src/main/java/org/apache/hudi/metrics/MetricsReporterType.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package com.uber.hoodie.metrics; +package org.apache.hudi.metrics; /** * Types of the reporter. Right now we only support Graphite. We can include JMX and CSV in the diff --git a/hoodie-client/src/main/java/com/uber/hoodie/table/HoodieCopyOnWriteTable.java b/hudi-client/src/main/java/org/apache/hudi/table/HoodieCopyOnWriteTable.java similarity index 95% rename from hoodie-client/src/main/java/com/uber/hoodie/table/HoodieCopyOnWriteTable.java rename to hudi-client/src/main/java/org/apache/hudi/table/HoodieCopyOnWriteTable.java index da04a311a..202d04720 100644 --- a/hoodie-client/src/main/java/com/uber/hoodie/table/HoodieCopyOnWriteTable.java +++ b/hudi-client/src/main/java/org/apache/hudi/table/HoodieCopyOnWriteTable.java @@ -16,39 +16,9 @@ * limitations under the License. */ -package com.uber.hoodie.table; +package org.apache.hudi.table; import com.google.common.hash.Hashing; -import com.uber.hoodie.WriteStatus; -import com.uber.hoodie.avro.model.HoodieCompactionPlan; -import com.uber.hoodie.common.HoodieCleanStat; -import com.uber.hoodie.common.HoodieRollbackStat; -import com.uber.hoodie.common.model.HoodieCommitMetadata; -import com.uber.hoodie.common.model.HoodieDataFile; -import com.uber.hoodie.common.model.HoodieKey; -import com.uber.hoodie.common.model.HoodieRecord; -import com.uber.hoodie.common.model.HoodieRecordLocation; -import com.uber.hoodie.common.model.HoodieRecordPayload; -import com.uber.hoodie.common.model.HoodieRollingStatMetadata; -import com.uber.hoodie.common.table.HoodieTimeline; -import com.uber.hoodie.common.table.timeline.HoodieActiveTimeline; -import com.uber.hoodie.common.table.timeline.HoodieInstant; -import com.uber.hoodie.common.util.FSUtils; -import com.uber.hoodie.common.util.Option; -import com.uber.hoodie.common.util.collection.Pair; -import com.uber.hoodie.common.util.queue.BoundedInMemoryExecutor; -import com.uber.hoodie.common.util.queue.BoundedInMemoryQueueConsumer; -import com.uber.hoodie.config.HoodieWriteConfig; -import com.uber.hoodie.exception.HoodieException; -import com.uber.hoodie.exception.HoodieIOException; -import com.uber.hoodie.exception.HoodieNotSupportedException; -import com.uber.hoodie.exception.HoodieUpsertException; -import com.uber.hoodie.func.CopyOnWriteLazyInsertIterable; -import com.uber.hoodie.func.ParquetReaderIterator; -import com.uber.hoodie.func.SparkBoundedInMemoryExecutor; -import com.uber.hoodie.io.HoodieCleanHelper; -import com.uber.hoodie.io.HoodieCreateHandle; -import com.uber.hoodie.io.HoodieMergeHandle; import java.io.IOException; import java.io.Serializable; import java.nio.charset.StandardCharsets; @@ -66,6 +36,36 @@ import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.PathFilter; +import org.apache.hudi.WriteStatus; +import org.apache.hudi.avro.model.HoodieCompactionPlan; +import org.apache.hudi.common.HoodieCleanStat; +import org.apache.hudi.common.HoodieRollbackStat; +import org.apache.hudi.common.model.HoodieCommitMetadata; +import org.apache.hudi.common.model.HoodieDataFile; +import org.apache.hudi.common.model.HoodieKey; +import org.apache.hudi.common.model.HoodieRecord; +import org.apache.hudi.common.model.HoodieRecordLocation; +import org.apache.hudi.common.model.HoodieRecordPayload; +import org.apache.hudi.common.model.HoodieRollingStatMetadata; +import org.apache.hudi.common.table.HoodieTimeline; +import org.apache.hudi.common.table.timeline.HoodieActiveTimeline; +import org.apache.hudi.common.table.timeline.HoodieInstant; +import org.apache.hudi.common.util.FSUtils; +import org.apache.hudi.common.util.Option; +import org.apache.hudi.common.util.collection.Pair; +import org.apache.hudi.common.util.queue.BoundedInMemoryExecutor; +import org.apache.hudi.common.util.queue.BoundedInMemoryQueueConsumer; +import org.apache.hudi.config.HoodieWriteConfig; +import org.apache.hudi.exception.HoodieException; +import org.apache.hudi.exception.HoodieIOException; +import org.apache.hudi.exception.HoodieNotSupportedException; +import org.apache.hudi.exception.HoodieUpsertException; +import org.apache.hudi.func.CopyOnWriteLazyInsertIterable; +import org.apache.hudi.func.ParquetReaderIterator; +import org.apache.hudi.func.SparkBoundedInMemoryExecutor; +import org.apache.hudi.io.HoodieCleanHelper; +import org.apache.hudi.io.HoodieCreateHandle; +import org.apache.hudi.io.HoodieMergeHandle; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; import org.apache.parquet.avro.AvroParquetReader; diff --git a/hoodie-client/src/main/java/com/uber/hoodie/table/HoodieMergeOnReadTable.java b/hudi-client/src/main/java/org/apache/hudi/table/HoodieMergeOnReadTable.java similarity index 93% rename from hoodie-client/src/main/java/com/uber/hoodie/table/HoodieMergeOnReadTable.java rename to hudi-client/src/main/java/org/apache/hudi/table/HoodieMergeOnReadTable.java index 97e8c7551..ad0424b18 100644 --- a/hoodie-client/src/main/java/com/uber/hoodie/table/HoodieMergeOnReadTable.java +++ b/hudi-client/src/main/java/org/apache/hudi/table/HoodieMergeOnReadTable.java @@ -16,43 +16,11 @@ * limitations under the License. */ -package com.uber.hoodie.table; +package org.apache.hudi.table; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.Maps; import com.google.common.collect.Sets; -import com.uber.hoodie.WriteStatus; -import com.uber.hoodie.avro.model.HoodieCompactionPlan; -import com.uber.hoodie.common.HoodieRollbackStat; -import com.uber.hoodie.common.model.FileSlice; -import com.uber.hoodie.common.model.HoodieCommitMetadata; -import com.uber.hoodie.common.model.HoodieLogFile; -import com.uber.hoodie.common.model.HoodieRecord; -import com.uber.hoodie.common.model.HoodieRecordLocation; -import com.uber.hoodie.common.model.HoodieRecordPayload; -import com.uber.hoodie.common.model.HoodieRollingStat; -import com.uber.hoodie.common.model.HoodieRollingStatMetadata; -import com.uber.hoodie.common.model.HoodieWriteStat; -import com.uber.hoodie.common.table.HoodieTimeline; -import com.uber.hoodie.common.table.SyncableFileSystemView; -import com.uber.hoodie.common.table.log.HoodieLogFormat; -import com.uber.hoodie.common.table.log.block.HoodieCommandBlock; -import com.uber.hoodie.common.table.log.block.HoodieCommandBlock.HoodieCommandBlockTypeEnum; -import com.uber.hoodie.common.table.log.block.HoodieLogBlock.HeaderMetadataType; -import com.uber.hoodie.common.table.timeline.HoodieActiveTimeline; -import com.uber.hoodie.common.table.timeline.HoodieInstant; -import com.uber.hoodie.common.util.FSUtils; -import com.uber.hoodie.common.util.Option; -import com.uber.hoodie.config.HoodieWriteConfig; -import com.uber.hoodie.exception.HoodieCompactionException; -import com.uber.hoodie.exception.HoodieException; -import com.uber.hoodie.exception.HoodieIOException; -import com.uber.hoodie.exception.HoodieRollbackException; -import com.uber.hoodie.exception.HoodieUpsertException; -import com.uber.hoodie.func.MergeOnReadLazyInsertIterable; -import com.uber.hoodie.index.HoodieIndex; -import com.uber.hoodie.io.HoodieAppendHandle; -import com.uber.hoodie.io.compact.HoodieRealtimeTableCompactor; import java.io.IOException; import java.io.UncheckedIOException; import java.util.ArrayList; @@ -67,6 +35,39 @@ import java.util.stream.Collectors; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.PathFilter; +import org.apache.hudi.WriteStatus; +import org.apache.hudi.avro.model.HoodieCompactionPlan; +import org.apache.hudi.common.HoodieRollbackStat; +import org.apache.hudi.common.model.FileSlice; +import org.apache.hudi.common.model.HoodieCommitMetadata; +import org.apache.hudi.common.model.HoodieLogFile; +import org.apache.hudi.common.model.HoodieRecord; +import org.apache.hudi.common.model.HoodieRecordLocation; +import org.apache.hudi.common.model.HoodieRecordPayload; +import org.apache.hudi.common.model.HoodieRollingStat; +import org.apache.hudi.common.model.HoodieRollingStatMetadata; +import org.apache.hudi.common.model.HoodieWriteStat; +import org.apache.hudi.common.table.HoodieTimeline; +import org.apache.hudi.common.table.SyncableFileSystemView; +import org.apache.hudi.common.table.log.HoodieLogFormat; +import org.apache.hudi.common.table.log.HoodieLogFormat.Writer; +import org.apache.hudi.common.table.log.block.HoodieCommandBlock; +import org.apache.hudi.common.table.log.block.HoodieCommandBlock.HoodieCommandBlockTypeEnum; +import org.apache.hudi.common.table.log.block.HoodieLogBlock.HeaderMetadataType; +import org.apache.hudi.common.table.timeline.HoodieActiveTimeline; +import org.apache.hudi.common.table.timeline.HoodieInstant; +import org.apache.hudi.common.util.FSUtils; +import org.apache.hudi.common.util.Option; +import org.apache.hudi.config.HoodieWriteConfig; +import org.apache.hudi.exception.HoodieCompactionException; +import org.apache.hudi.exception.HoodieException; +import org.apache.hudi.exception.HoodieIOException; +import org.apache.hudi.exception.HoodieRollbackException; +import org.apache.hudi.exception.HoodieUpsertException; +import org.apache.hudi.func.MergeOnReadLazyInsertIterable; +import org.apache.hudi.index.HoodieIndex; +import org.apache.hudi.io.HoodieAppendHandle; +import org.apache.hudi.io.compact.HoodieRealtimeTableCompactor; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; import org.apache.spark.Partitioner; @@ -503,7 +504,7 @@ public class HoodieMergeOnReadTable extends return wStat != null && wStat.getPrevCommit() != HoodieWriteStat.NULL_COMMIT && wStat.getPrevCommit() != null && !deletedFiles.contains(wStat.getFileId()); }).forEach(wStat -> { - HoodieLogFormat.Writer writer = null; + Writer writer = null; String baseCommitTime = fileIdToBaseCommitTimeForLogMap.get(wStat.getFileId()); boolean success = false; try { diff --git a/hoodie-client/src/main/java/com/uber/hoodie/table/HoodieTable.java b/hudi-client/src/main/java/org/apache/hudi/table/HoodieTable.java similarity index 90% rename from hoodie-client/src/main/java/com/uber/hoodie/table/HoodieTable.java rename to hudi-client/src/main/java/org/apache/hudi/table/HoodieTable.java index 7fd9a29d4..0ac18fcd6 100644 --- a/hoodie-client/src/main/java/com/uber/hoodie/table/HoodieTable.java +++ b/hudi-client/src/main/java/org/apache/hudi/table/HoodieTable.java @@ -16,37 +16,8 @@ * limitations under the License. */ -package com.uber.hoodie.table; +package org.apache.hudi.table; -import com.uber.hoodie.WriteStatus; -import com.uber.hoodie.avro.model.HoodieCompactionPlan; -import com.uber.hoodie.avro.model.HoodieSavepointMetadata; -import com.uber.hoodie.client.utils.ClientUtils; -import com.uber.hoodie.common.HoodieCleanStat; -import com.uber.hoodie.common.HoodieRollbackStat; -import com.uber.hoodie.common.SerializableConfiguration; -import com.uber.hoodie.common.model.HoodieRecord; -import com.uber.hoodie.common.model.HoodieRecordPayload; -import com.uber.hoodie.common.model.HoodieWriteStat; -import com.uber.hoodie.common.table.HoodieTableMetaClient; -import com.uber.hoodie.common.table.HoodieTimeline; -import com.uber.hoodie.common.table.SyncableFileSystemView; -import com.uber.hoodie.common.table.TableFileSystemView; -import com.uber.hoodie.common.table.timeline.HoodieActiveTimeline; -import com.uber.hoodie.common.table.timeline.HoodieInstant; -import com.uber.hoodie.common.table.view.FileSystemViewManager; -import com.uber.hoodie.common.table.view.HoodieTableFileSystemView; -import com.uber.hoodie.common.util.AvroUtils; -import com.uber.hoodie.common.util.ConsistencyGuard; -import com.uber.hoodie.common.util.ConsistencyGuard.FileVisibility; -import com.uber.hoodie.common.util.FSUtils; -import com.uber.hoodie.common.util.FailSafeConsistencyGuard; -import com.uber.hoodie.common.util.collection.Pair; -import com.uber.hoodie.config.HoodieWriteConfig; -import com.uber.hoodie.exception.HoodieException; -import com.uber.hoodie.exception.HoodieIOException; -import com.uber.hoodie.exception.HoodieSavepointException; -import com.uber.hoodie.index.HoodieIndex; import java.io.IOException; import java.io.Serializable; import java.util.ArrayList; @@ -59,6 +30,35 @@ import java.util.stream.Stream; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hudi.WriteStatus; +import org.apache.hudi.avro.model.HoodieCompactionPlan; +import org.apache.hudi.avro.model.HoodieSavepointMetadata; +import org.apache.hudi.client.utils.ClientUtils; +import org.apache.hudi.common.HoodieCleanStat; +import org.apache.hudi.common.HoodieRollbackStat; +import org.apache.hudi.common.SerializableConfiguration; +import org.apache.hudi.common.model.HoodieRecord; +import org.apache.hudi.common.model.HoodieRecordPayload; +import org.apache.hudi.common.model.HoodieWriteStat; +import org.apache.hudi.common.table.HoodieTableMetaClient; +import org.apache.hudi.common.table.HoodieTimeline; +import org.apache.hudi.common.table.SyncableFileSystemView; +import org.apache.hudi.common.table.TableFileSystemView; +import org.apache.hudi.common.table.timeline.HoodieActiveTimeline; +import org.apache.hudi.common.table.timeline.HoodieInstant; +import org.apache.hudi.common.table.view.FileSystemViewManager; +import org.apache.hudi.common.table.view.HoodieTableFileSystemView; +import org.apache.hudi.common.util.AvroUtils; +import org.apache.hudi.common.util.ConsistencyGuard; +import org.apache.hudi.common.util.ConsistencyGuard.FileVisibility; +import org.apache.hudi.common.util.FSUtils; +import org.apache.hudi.common.util.FailSafeConsistencyGuard; +import org.apache.hudi.common.util.collection.Pair; +import org.apache.hudi.config.HoodieWriteConfig; +import org.apache.hudi.exception.HoodieException; +import org.apache.hudi.exception.HoodieIOException; +import org.apache.hudi.exception.HoodieSavepointException; +import org.apache.hudi.index.HoodieIndex; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; import org.apache.spark.Partitioner; diff --git a/hoodie-client/src/main/java/com/uber/hoodie/table/UserDefinedBulkInsertPartitioner.java b/hudi-client/src/main/java/org/apache/hudi/table/UserDefinedBulkInsertPartitioner.java similarity index 91% rename from hoodie-client/src/main/java/com/uber/hoodie/table/UserDefinedBulkInsertPartitioner.java rename to hudi-client/src/main/java/org/apache/hudi/table/UserDefinedBulkInsertPartitioner.java index 2f6a70c70..6d6f18e9d 100644 --- a/hoodie-client/src/main/java/com/uber/hoodie/table/UserDefinedBulkInsertPartitioner.java +++ b/hudi-client/src/main/java/org/apache/hudi/table/UserDefinedBulkInsertPartitioner.java @@ -16,10 +16,10 @@ * limitations under the License. */ -package com.uber.hoodie.table; +package org.apache.hudi.table; -import com.uber.hoodie.common.model.HoodieRecord; -import com.uber.hoodie.common.model.HoodieRecordPayload; +import org.apache.hudi.common.model.HoodieRecord; +import org.apache.hudi.common.model.HoodieRecordPayload; import org.apache.spark.api.java.JavaRDD; /** diff --git a/hoodie-client/src/main/java/com/uber/hoodie/table/WorkloadProfile.java b/hudi-client/src/main/java/org/apache/hudi/table/WorkloadProfile.java similarity index 93% rename from hoodie-client/src/main/java/com/uber/hoodie/table/WorkloadProfile.java rename to hudi-client/src/main/java/org/apache/hudi/table/WorkloadProfile.java index 0975048f5..0ce4c62c3 100644 --- a/hoodie-client/src/main/java/com/uber/hoodie/table/WorkloadProfile.java +++ b/hudi-client/src/main/java/org/apache/hudi/table/WorkloadProfile.java @@ -16,16 +16,16 @@ * limitations under the License. */ -package com.uber.hoodie.table; +package org.apache.hudi.table; -import com.uber.hoodie.common.model.HoodieRecord; -import com.uber.hoodie.common.model.HoodieRecordLocation; -import com.uber.hoodie.common.model.HoodieRecordPayload; -import com.uber.hoodie.common.util.Option; import java.io.Serializable; import java.util.HashMap; import java.util.Map; import java.util.Set; +import org.apache.hudi.common.model.HoodieRecord; +import org.apache.hudi.common.model.HoodieRecordLocation; +import org.apache.hudi.common.model.HoodieRecordPayload; +import org.apache.hudi.common.util.Option; import org.apache.spark.api.java.JavaRDD; import scala.Tuple2; diff --git a/hoodie-client/src/main/java/com/uber/hoodie/table/WorkloadStat.java b/hudi-client/src/main/java/org/apache/hudi/table/WorkloadStat.java similarity index 93% rename from hoodie-client/src/main/java/com/uber/hoodie/table/WorkloadStat.java rename to hudi-client/src/main/java/org/apache/hudi/table/WorkloadStat.java index a613f3b61..ae759fa43 100644 --- a/hoodie-client/src/main/java/com/uber/hoodie/table/WorkloadStat.java +++ b/hudi-client/src/main/java/org/apache/hudi/table/WorkloadStat.java @@ -16,12 +16,12 @@ * limitations under the License. */ -package com.uber.hoodie.table; +package org.apache.hudi.table; -import com.uber.hoodie.common.model.HoodieRecordLocation; -import com.uber.hoodie.common.util.collection.Pair; import java.io.Serializable; import java.util.HashMap; +import org.apache.hudi.common.model.HoodieRecordLocation; +import org.apache.hudi.common.util.collection.Pair; /** * Wraps stats about a single partition path. diff --git a/hoodie-client/src/main/resources/META-INF/LICENSE.txt b/hudi-client/src/main/resources/META-INF/LICENSE.txt similarity index 100% rename from hoodie-client/src/main/resources/META-INF/LICENSE.txt rename to hudi-client/src/main/resources/META-INF/LICENSE.txt diff --git a/hoodie-client/src/main/resources/META-INF/NOTICE.txt b/hudi-client/src/main/resources/META-INF/NOTICE.txt similarity index 100% rename from hoodie-client/src/main/resources/META-INF/NOTICE.txt rename to hudi-client/src/main/resources/META-INF/NOTICE.txt diff --git a/hoodie-client/src/main/resources/log4j.properties b/hudi-client/src/main/resources/log4j.properties similarity index 100% rename from hoodie-client/src/main/resources/log4j.properties rename to hudi-client/src/main/resources/log4j.properties diff --git a/hoodie-client/src/test/java/HoodieClientExample.java b/hudi-client/src/test/java/HoodieClientExample.java similarity index 87% rename from hoodie-client/src/test/java/HoodieClientExample.java rename to hudi-client/src/test/java/HoodieClientExample.java index 44787ed35..7e298332a 100644 --- a/hoodie-client/src/test/java/HoodieClientExample.java +++ b/hudi-client/src/test/java/HoodieClientExample.java @@ -18,22 +18,22 @@ import com.beust.jcommander.JCommander; import com.beust.jcommander.Parameter; -import com.uber.hoodie.HoodieWriteClient; -import com.uber.hoodie.WriteStatus; -import com.uber.hoodie.common.HoodieTestDataGenerator; -import com.uber.hoodie.common.model.HoodieAvroPayload; -import com.uber.hoodie.common.model.HoodieRecord; -import com.uber.hoodie.common.model.HoodieTableType; -import com.uber.hoodie.common.table.HoodieTableMetaClient; -import com.uber.hoodie.common.util.FSUtils; -import com.uber.hoodie.common.util.Option; -import com.uber.hoodie.config.HoodieCompactionConfig; -import com.uber.hoodie.config.HoodieIndexConfig; -import com.uber.hoodie.config.HoodieWriteConfig; -import com.uber.hoodie.index.HoodieIndex.IndexType; import java.util.List; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hudi.HoodieWriteClient; +import org.apache.hudi.WriteStatus; +import org.apache.hudi.common.HoodieTestDataGenerator; +import org.apache.hudi.common.model.HoodieAvroPayload; +import org.apache.hudi.common.model.HoodieRecord; +import org.apache.hudi.common.model.HoodieTableType; +import org.apache.hudi.common.table.HoodieTableMetaClient; +import org.apache.hudi.common.util.FSUtils; +import org.apache.hudi.common.util.Option; +import org.apache.hudi.config.HoodieCompactionConfig; +import org.apache.hudi.config.HoodieIndexConfig; +import org.apache.hudi.config.HoodieWriteConfig; +import org.apache.hudi.index.HoodieIndex.IndexType; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; import org.apache.spark.SparkConf; diff --git a/hoodie-client/src/test/java/com/uber/hoodie/TestAsyncCompaction.java b/hudi-client/src/test/java/org/apache/hudi/TestAsyncCompaction.java similarity index 93% rename from hoodie-client/src/test/java/com/uber/hoodie/TestAsyncCompaction.java rename to hudi-client/src/test/java/org/apache/hudi/TestAsyncCompaction.java index d7da020e9..9dc9a1854 100644 --- a/hoodie-client/src/test/java/com/uber/hoodie/TestAsyncCompaction.java +++ b/hudi-client/src/test/java/org/apache/hudi/TestAsyncCompaction.java @@ -16,40 +16,13 @@ * limitations under the License. */ -package com.uber.hoodie; +package org.apache.hudi; -import static com.uber.hoodie.common.HoodieTestDataGenerator.TRIP_EXAMPLE_SCHEMA; +import static org.apache.hudi.common.HoodieTestDataGenerator.TRIP_EXAMPLE_SCHEMA; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; -import com.uber.hoodie.avro.model.HoodieCompactionOperation; -import com.uber.hoodie.avro.model.HoodieCompactionPlan; -import com.uber.hoodie.common.HoodieClientTestUtils; -import com.uber.hoodie.common.HoodieTestDataGenerator; -import com.uber.hoodie.common.model.FileSlice; -import com.uber.hoodie.common.model.HoodieDataFile; -import com.uber.hoodie.common.model.HoodieFileGroupId; -import com.uber.hoodie.common.model.HoodieRecord; -import com.uber.hoodie.common.model.HoodieTableType; -import com.uber.hoodie.common.model.HoodieTestUtils; -import com.uber.hoodie.common.table.HoodieTableMetaClient; -import com.uber.hoodie.common.table.HoodieTimeline; -import com.uber.hoodie.common.table.timeline.HoodieInstant; -import com.uber.hoodie.common.table.timeline.HoodieInstant.State; -import com.uber.hoodie.common.table.view.FileSystemViewStorageConfig; -import com.uber.hoodie.common.table.view.FileSystemViewStorageType; -import com.uber.hoodie.common.table.view.HoodieTableFileSystemView; -import com.uber.hoodie.common.util.AvroUtils; -import com.uber.hoodie.common.util.CompactionUtils; -import com.uber.hoodie.common.util.Option; -import com.uber.hoodie.common.util.collection.Pair; -import com.uber.hoodie.config.HoodieCompactionConfig; -import com.uber.hoodie.config.HoodieIndexConfig; -import com.uber.hoodie.config.HoodieStorageConfig; -import com.uber.hoodie.config.HoodieWriteConfig; -import com.uber.hoodie.index.HoodieIndex; -import com.uber.hoodie.table.HoodieTable; import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; @@ -58,7 +31,35 @@ import java.util.Map; import java.util.stream.Collectors; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.Path; +import org.apache.hudi.avro.model.HoodieCompactionOperation; +import org.apache.hudi.avro.model.HoodieCompactionPlan; +import org.apache.hudi.common.HoodieClientTestUtils; +import org.apache.hudi.common.HoodieTestDataGenerator; +import org.apache.hudi.common.model.FileSlice; +import org.apache.hudi.common.model.HoodieDataFile; +import org.apache.hudi.common.model.HoodieFileGroupId; +import org.apache.hudi.common.model.HoodieRecord; +import org.apache.hudi.common.model.HoodieTableType; +import org.apache.hudi.common.model.HoodieTestUtils; +import org.apache.hudi.common.table.HoodieTableMetaClient; +import org.apache.hudi.common.table.HoodieTimeline; +import org.apache.hudi.common.table.timeline.HoodieInstant; +import org.apache.hudi.common.table.timeline.HoodieInstant.State; +import org.apache.hudi.common.table.view.FileSystemViewStorageConfig; +import org.apache.hudi.common.table.view.FileSystemViewStorageType; +import org.apache.hudi.common.table.view.HoodieTableFileSystemView; +import org.apache.hudi.common.util.AvroUtils; +import org.apache.hudi.common.util.CompactionUtils; +import org.apache.hudi.common.util.Option; +import org.apache.hudi.common.util.collection.Pair; +import org.apache.hudi.config.HoodieCompactionConfig; +import org.apache.hudi.config.HoodieIndexConfig; +import org.apache.hudi.config.HoodieStorageConfig; +import org.apache.hudi.config.HoodieWriteConfig; +import org.apache.hudi.index.HoodieIndex; +import org.apache.hudi.table.HoodieTable; import org.apache.spark.api.java.JavaRDD; +import org.junit.Assert; import org.junit.Test; /** @@ -452,7 +453,7 @@ public class TestAsyncCompaction extends TestHoodieClientBase { if (!cfg.shouldAutoCommit()) { client.commit(firstInstant, statuses); } - TestHoodieClientBase.assertNoWriteErrors(statusList); + assertNoWriteErrors(statusList); metaClient = new HoodieTableMetaClient(jsc.hadoopConfiguration(), cfg.getBasePath()); HoodieTable hoodieTable = getHoodieTable(metaClient, cfg); List dataFilesToRead = getCurrentLatestDataFiles(hoodieTable, cfg); @@ -527,7 +528,7 @@ public class TestAsyncCompaction extends TestHoodieClientBase { String latestCompactionCommitTime = timeline.lastInstant().get().getTimestamp(); assertEquals("Expect compaction instant time to be the latest commit time", latestCompactionCommitTime, compactionInstantTime); - assertEquals("Must contain expected records", expectedNumRecs, + Assert.assertEquals("Must contain expected records", expectedNumRecs, HoodieClientTestUtils.readSince(basePath, sqlContext, timeline, "000").count()); } @@ -540,7 +541,7 @@ public class TestAsyncCompaction extends TestHoodieClientBase { JavaRDD statuses = client.upsert(writeRecords, instantTime); List statusList = statuses.collect(); - TestHoodieClientBase.assertNoWriteErrors(statusList); + assertNoWriteErrors(statusList); if (!cfg.shouldAutoCommit() && !skipCommit) { client.commit(instantTime, statuses); } diff --git a/hoodie-client/src/test/java/com/uber/hoodie/TestCleaner.java b/hudi-client/src/test/java/org/apache/hudi/TestCleaner.java similarity index 70% rename from hoodie-client/src/test/java/com/uber/hoodie/TestCleaner.java rename to hudi-client/src/test/java/org/apache/hudi/TestCleaner.java index ff78494a4..7ef12400a 100644 --- a/hoodie-client/src/test/java/com/uber/hoodie/TestCleaner.java +++ b/hudi-client/src/test/java/org/apache/hudi/TestCleaner.java @@ -16,46 +16,13 @@ * limitations under the License. */ -package com.uber.hoodie; +package org.apache.hudi; -import static com.uber.hoodie.common.HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH; -import static com.uber.hoodie.common.HoodieTestDataGenerator.DEFAULT_SECOND_PARTITION_PATH; -import static com.uber.hoodie.common.HoodieTestDataGenerator.DEFAULT_THIRD_PARTITION_PATH; -import static com.uber.hoodie.common.table.HoodieTimeline.COMPACTION_ACTION; -import static com.uber.hoodie.common.table.HoodieTimeline.GREATER; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; import com.google.common.collect.Iterables; -import com.uber.hoodie.avro.model.HoodieCompactionPlan; -import com.uber.hoodie.common.HoodieCleanStat; -import com.uber.hoodie.common.model.FileSlice; -import com.uber.hoodie.common.model.HoodieCleaningPolicy; -import com.uber.hoodie.common.model.HoodieCommitMetadata; -import com.uber.hoodie.common.model.HoodieDataFile; -import com.uber.hoodie.common.model.HoodieFileGroup; -import com.uber.hoodie.common.model.HoodieFileGroupId; -import com.uber.hoodie.common.model.HoodieRecord; -import com.uber.hoodie.common.model.HoodieTableType; -import com.uber.hoodie.common.model.HoodieTestUtils; -import com.uber.hoodie.common.model.HoodieWriteStat; -import com.uber.hoodie.common.table.HoodieTableMetaClient; -import com.uber.hoodie.common.table.HoodieTimeline; -import com.uber.hoodie.common.table.TableFileSystemView; -import com.uber.hoodie.common.table.timeline.HoodieActiveTimeline; -import com.uber.hoodie.common.table.timeline.HoodieInstant; -import com.uber.hoodie.common.table.timeline.HoodieInstant.State; -import com.uber.hoodie.common.util.AvroUtils; -import com.uber.hoodie.common.util.CompactionUtils; -import com.uber.hoodie.common.util.ConsistencyGuardConfig; -import com.uber.hoodie.common.util.FSUtils; -import com.uber.hoodie.common.util.Option; -import com.uber.hoodie.common.util.collection.Pair; -import com.uber.hoodie.config.HoodieCompactionConfig; -import com.uber.hoodie.config.HoodieWriteConfig; -import com.uber.hoodie.index.HoodieIndex; -import com.uber.hoodie.table.HoodieTable; import java.io.IOException; import java.nio.file.Paths; import java.util.ArrayList; @@ -70,6 +37,35 @@ import java.util.stream.Collectors; import java.util.stream.Stream; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.RemoteIterator; +import org.apache.hudi.avro.model.HoodieCompactionPlan; +import org.apache.hudi.common.HoodieCleanStat; +import org.apache.hudi.common.HoodieTestDataGenerator; +import org.apache.hudi.common.model.FileSlice; +import org.apache.hudi.common.model.HoodieCleaningPolicy; +import org.apache.hudi.common.model.HoodieCommitMetadata; +import org.apache.hudi.common.model.HoodieDataFile; +import org.apache.hudi.common.model.HoodieFileGroup; +import org.apache.hudi.common.model.HoodieFileGroupId; +import org.apache.hudi.common.model.HoodieRecord; +import org.apache.hudi.common.model.HoodieTableType; +import org.apache.hudi.common.model.HoodieTestUtils; +import org.apache.hudi.common.model.HoodieWriteStat; +import org.apache.hudi.common.table.HoodieTableMetaClient; +import org.apache.hudi.common.table.HoodieTimeline; +import org.apache.hudi.common.table.TableFileSystemView; +import org.apache.hudi.common.table.timeline.HoodieActiveTimeline; +import org.apache.hudi.common.table.timeline.HoodieInstant; +import org.apache.hudi.common.table.timeline.HoodieInstant.State; +import org.apache.hudi.common.util.AvroUtils; +import org.apache.hudi.common.util.CompactionUtils; +import org.apache.hudi.common.util.ConsistencyGuardConfig; +import org.apache.hudi.common.util.FSUtils; +import org.apache.hudi.common.util.Option; +import org.apache.hudi.common.util.collection.Pair; +import org.apache.hudi.config.HoodieCompactionConfig; +import org.apache.hudi.config.HoodieWriteConfig; +import org.apache.hudi.index.HoodieIndex; +import org.apache.hudi.table.HoodieTable; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; import org.apache.spark.api.java.JavaRDD; @@ -232,7 +228,7 @@ public class TestCleaner extends TestHoodieClientBase { List instantTimes = HoodieTestUtils.monotonicIncreasingCommitTimestamps(9, 1); String compactionTime = instantTimes.get(0); table.getActiveTimeline().saveToCompactionRequested( - new HoodieInstant(State.REQUESTED, COMPACTION_ACTION, compactionTime), + new HoodieInstant(State.REQUESTED, HoodieTimeline.COMPACTION_ACTION, compactionTime), AvroUtils.serializeCompactionPlan(compactionPlan)); instantTimes = instantTimes.subList(1, instantTimes.size()); @@ -427,62 +423,88 @@ public class TestCleaner extends TestHoodieClientBase { // make 1 commit, with 1 file per partition HoodieTestUtils.createCommitFiles(basePath, "000"); - String file1P0C0 = HoodieTestUtils.createNewDataFile(basePath, DEFAULT_FIRST_PARTITION_PATH, "000"); - String file1P1C0 = HoodieTestUtils.createNewDataFile(basePath, DEFAULT_SECOND_PARTITION_PATH, "000"); + String file1P0C0 = HoodieTestUtils + .createNewDataFile(basePath, HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH, "000"); + String file1P1C0 = HoodieTestUtils + .createNewDataFile(basePath, HoodieTestDataGenerator.DEFAULT_SECOND_PARTITION_PATH, "000"); HoodieTable table = HoodieTable.getHoodieTable( new HoodieTableMetaClient(jsc.hadoopConfiguration(), config.getBasePath(), true), config, jsc); List hoodieCleanStatsOne = table.clean(jsc); assertEquals("Must not clean any files", 0, - getCleanStat(hoodieCleanStatsOne, DEFAULT_FIRST_PARTITION_PATH).getSuccessDeleteFiles().size()); + getCleanStat(hoodieCleanStatsOne, HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH).getSuccessDeleteFiles() + .size()); assertEquals("Must not clean any files", 0, - getCleanStat(hoodieCleanStatsOne, DEFAULT_SECOND_PARTITION_PATH).getSuccessDeleteFiles().size()); - assertTrue(HoodieTestUtils.doesDataFileExist(basePath, DEFAULT_FIRST_PARTITION_PATH, "000", file1P0C0)); - assertTrue(HoodieTestUtils.doesDataFileExist(basePath, DEFAULT_SECOND_PARTITION_PATH, "000", file1P1C0)); + getCleanStat(hoodieCleanStatsOne, HoodieTestDataGenerator.DEFAULT_SECOND_PARTITION_PATH).getSuccessDeleteFiles() + .size()); + assertTrue(HoodieTestUtils + .doesDataFileExist(basePath, HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH, "000", file1P0C0)); + assertTrue(HoodieTestUtils + .doesDataFileExist(basePath, HoodieTestDataGenerator.DEFAULT_SECOND_PARTITION_PATH, "000", file1P1C0)); // make next commit, with 1 insert & 1 update per partition HoodieTestUtils.createCommitFiles(basePath, "001"); table = HoodieTable.getHoodieTable(new HoodieTableMetaClient(jsc.hadoopConfiguration(), basePath, true), config, jsc); - String file2P0C1 = HoodieTestUtils.createNewDataFile(basePath, DEFAULT_FIRST_PARTITION_PATH, "001"); // insert - String file2P1C1 = HoodieTestUtils.createNewDataFile(basePath, DEFAULT_SECOND_PARTITION_PATH, "001"); // insert - HoodieTestUtils.createDataFile(basePath, DEFAULT_FIRST_PARTITION_PATH, "001", file1P0C0); // update - HoodieTestUtils.createDataFile(basePath, DEFAULT_SECOND_PARTITION_PATH, "001", file1P1C0); // update + String file2P0C1 = HoodieTestUtils + .createNewDataFile(basePath, HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH, "001"); // insert + String file2P1C1 = HoodieTestUtils + .createNewDataFile(basePath, HoodieTestDataGenerator.DEFAULT_SECOND_PARTITION_PATH, "001"); // insert + HoodieTestUtils + .createDataFile(basePath, HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH, "001", file1P0C0); // update + HoodieTestUtils + .createDataFile(basePath, HoodieTestDataGenerator.DEFAULT_SECOND_PARTITION_PATH, "001", file1P1C0); // update List hoodieCleanStatsTwo = table.clean(jsc); assertEquals("Must clean 1 file", 1, - getCleanStat(hoodieCleanStatsTwo, DEFAULT_FIRST_PARTITION_PATH).getSuccessDeleteFiles().size()); + getCleanStat(hoodieCleanStatsTwo, HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH).getSuccessDeleteFiles() + .size()); assertEquals("Must clean 1 file", 1, - getCleanStat(hoodieCleanStatsTwo, DEFAULT_SECOND_PARTITION_PATH).getSuccessDeleteFiles().size()); - assertTrue(HoodieTestUtils.doesDataFileExist(basePath, DEFAULT_FIRST_PARTITION_PATH, "001", file2P0C1)); - assertTrue(HoodieTestUtils.doesDataFileExist(basePath, DEFAULT_SECOND_PARTITION_PATH, "001", file2P1C1)); - assertFalse(HoodieTestUtils.doesDataFileExist(basePath, DEFAULT_FIRST_PARTITION_PATH, "000", file1P0C0)); - assertFalse(HoodieTestUtils.doesDataFileExist(basePath, DEFAULT_SECOND_PARTITION_PATH, "000", file1P1C0)); + getCleanStat(hoodieCleanStatsTwo, HoodieTestDataGenerator.DEFAULT_SECOND_PARTITION_PATH).getSuccessDeleteFiles() + .size()); + assertTrue(HoodieTestUtils + .doesDataFileExist(basePath, HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH, "001", file2P0C1)); + assertTrue(HoodieTestUtils + .doesDataFileExist(basePath, HoodieTestDataGenerator.DEFAULT_SECOND_PARTITION_PATH, "001", file2P1C1)); + assertFalse(HoodieTestUtils + .doesDataFileExist(basePath, HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH, "000", file1P0C0)); + assertFalse(HoodieTestUtils + .doesDataFileExist(basePath, HoodieTestDataGenerator.DEFAULT_SECOND_PARTITION_PATH, "000", file1P1C0)); // make next commit, with 2 updates to existing files, and 1 insert HoodieTestUtils.createCommitFiles(basePath, "002"); table = HoodieTable.getHoodieTable(new HoodieTableMetaClient(jsc.hadoopConfiguration(), config.getBasePath(), true), config, jsc); - HoodieTestUtils.createDataFile(basePath, DEFAULT_FIRST_PARTITION_PATH, "002", file1P0C0); // update - HoodieTestUtils.createDataFile(basePath, DEFAULT_FIRST_PARTITION_PATH, "002", file2P0C1); // update - String file3P0C2 = HoodieTestUtils.createNewDataFile(basePath, DEFAULT_FIRST_PARTITION_PATH, "002"); + HoodieTestUtils + .createDataFile(basePath, HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH, "002", file1P0C0); // update + HoodieTestUtils + .createDataFile(basePath, HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH, "002", file2P0C1); // update + String file3P0C2 = HoodieTestUtils + .createNewDataFile(basePath, HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH, "002"); List hoodieCleanStatsThree = table.clean(jsc); assertEquals("Must clean two files", 2, - getCleanStat(hoodieCleanStatsThree, DEFAULT_FIRST_PARTITION_PATH).getSuccessDeleteFiles().size()); - assertFalse(HoodieTestUtils.doesDataFileExist(basePath, DEFAULT_FIRST_PARTITION_PATH, "001", file1P0C0)); - assertFalse(HoodieTestUtils.doesDataFileExist(basePath, DEFAULT_FIRST_PARTITION_PATH, "001", file2P0C1)); - assertTrue(HoodieTestUtils.doesDataFileExist(basePath, DEFAULT_FIRST_PARTITION_PATH, "002", file3P0C2)); + getCleanStat(hoodieCleanStatsThree, HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH) + .getSuccessDeleteFiles().size()); + assertFalse(HoodieTestUtils + .doesDataFileExist(basePath, HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH, "001", file1P0C0)); + assertFalse(HoodieTestUtils + .doesDataFileExist(basePath, HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH, "001", file2P0C1)); + assertTrue(HoodieTestUtils + .doesDataFileExist(basePath, HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH, "002", file3P0C2)); // No cleaning on partially written file, with no commit. - HoodieTestUtils.createDataFile(basePath, DEFAULT_FIRST_PARTITION_PATH, "003", file3P0C2); // update + HoodieTestUtils + .createDataFile(basePath, HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH, "003", file3P0C2); // update List hoodieCleanStatsFour = table.clean(jsc); assertEquals("Must not clean any files", 0, - getCleanStat(hoodieCleanStatsFour, DEFAULT_FIRST_PARTITION_PATH).getSuccessDeleteFiles().size()); - assertTrue(HoodieTestUtils.doesDataFileExist(basePath, DEFAULT_FIRST_PARTITION_PATH, "002", file3P0C2)); + getCleanStat(hoodieCleanStatsFour, HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH).getSuccessDeleteFiles() + .size()); + assertTrue(HoodieTestUtils + .doesDataFileExist(basePath, HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH, "002", file3P0C2)); } /** @@ -500,34 +522,46 @@ public class TestCleaner extends TestHoodieClientBase { HoodieTableType.MERGE_ON_READ); // Make 3 files, one base file and 2 log files associated with base file - String file1P0 = HoodieTestUtils.createNewDataFile(basePath, DEFAULT_FIRST_PARTITION_PATH, "000"); + String file1P0 = HoodieTestUtils + .createNewDataFile(basePath, HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH, "000"); String file2P0L0 = HoodieTestUtils - .createNewLogFile(fs, basePath, DEFAULT_FIRST_PARTITION_PATH, "000", file1P0, Option.empty()); + .createNewLogFile(fs, basePath, HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH, "000", file1P0, + Option.empty()); String file2P0L1 = HoodieTestUtils - .createNewLogFile(fs, basePath, DEFAULT_FIRST_PARTITION_PATH, "000", file1P0, Option.of(2)); + .createNewLogFile(fs, basePath, HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH, "000", file1P0, + Option.of(2)); // make 1 compaction commit HoodieTestUtils.createCompactionCommitFiles(fs, basePath, "000"); // Make 4 files, one base file and 3 log files associated with base file - HoodieTestUtils.createDataFile(basePath, DEFAULT_FIRST_PARTITION_PATH, "001", file1P0); + HoodieTestUtils.createDataFile(basePath, HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH, "001", file1P0); file2P0L0 = HoodieTestUtils - .createNewLogFile(fs, basePath, DEFAULT_FIRST_PARTITION_PATH, "001", file1P0, Option.empty()); + .createNewLogFile(fs, basePath, HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH, "001", file1P0, + Option.empty()); file2P0L0 = HoodieTestUtils - .createNewLogFile(fs, basePath, DEFAULT_FIRST_PARTITION_PATH, "001", file1P0, Option.of(2)); + .createNewLogFile(fs, basePath, HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH, "001", file1P0, + Option.of(2)); file2P0L0 = HoodieTestUtils - .createNewLogFile(fs, basePath, DEFAULT_FIRST_PARTITION_PATH, "001", file1P0, Option.of(3)); + .createNewLogFile(fs, basePath, HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH, "001", file1P0, + Option.of(3)); // make 1 compaction commit HoodieTestUtils.createCompactionCommitFiles(fs, basePath, "001"); HoodieTable table = HoodieTable.getHoodieTable(metaClient, config, jsc); List hoodieCleanStats = table.clean(jsc); assertEquals("Must clean three files, one parquet and 2 log files", 3, - getCleanStat(hoodieCleanStats, DEFAULT_FIRST_PARTITION_PATH).getSuccessDeleteFiles().size()); - assertFalse(HoodieTestUtils.doesDataFileExist(basePath, DEFAULT_FIRST_PARTITION_PATH, "000", file1P0)); + getCleanStat(hoodieCleanStats, HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH).getSuccessDeleteFiles() + .size()); + assertFalse(HoodieTestUtils + .doesDataFileExist(basePath, HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH, "000", file1P0)); assertFalse( - HoodieTestUtils.doesLogFileExist(basePath, DEFAULT_FIRST_PARTITION_PATH, "000", file2P0L0, Option.empty())); + HoodieTestUtils + .doesLogFileExist(basePath, HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH, "000", file2P0L0, + Option.empty())); assertFalse( - HoodieTestUtils.doesLogFileExist(basePath, DEFAULT_FIRST_PARTITION_PATH, "000", file2P0L0, Option.of(2))); + HoodieTestUtils + .doesLogFileExist(basePath, HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH, "000", file2P0L0, + Option.of(2))); } /** @@ -542,8 +576,10 @@ public class TestCleaner extends TestHoodieClientBase { // make 1 commit, with 1 file per partition HoodieTestUtils.createCommitFiles(basePath, "000"); - String file1P0C0 = HoodieTestUtils.createNewDataFile(basePath, DEFAULT_FIRST_PARTITION_PATH, "000"); - String file1P1C0 = HoodieTestUtils.createNewDataFile(basePath, DEFAULT_SECOND_PARTITION_PATH, "000"); + String file1P0C0 = HoodieTestUtils + .createNewDataFile(basePath, HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH, "000"); + String file1P1C0 = HoodieTestUtils + .createNewDataFile(basePath, HoodieTestDataGenerator.DEFAULT_SECOND_PARTITION_PATH, "000"); HoodieTable table = HoodieTable.getHoodieTable( new HoodieTableMetaClient(jsc.hadoopConfiguration(), config.getBasePath(), true), config, @@ -551,75 +587,109 @@ public class TestCleaner extends TestHoodieClientBase { List hoodieCleanStatsOne = table.clean(jsc); assertEquals("Must not clean any files", 0, - getCleanStat(hoodieCleanStatsOne, DEFAULT_FIRST_PARTITION_PATH).getSuccessDeleteFiles().size()); + getCleanStat(hoodieCleanStatsOne, HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH).getSuccessDeleteFiles() + .size()); assertEquals("Must not clean any files", 0, - getCleanStat(hoodieCleanStatsOne, DEFAULT_SECOND_PARTITION_PATH).getSuccessDeleteFiles().size()); - assertTrue(HoodieTestUtils.doesDataFileExist(basePath, DEFAULT_FIRST_PARTITION_PATH, "000", file1P0C0)); - assertTrue(HoodieTestUtils.doesDataFileExist(basePath, DEFAULT_SECOND_PARTITION_PATH, "000", file1P1C0)); + getCleanStat(hoodieCleanStatsOne, HoodieTestDataGenerator.DEFAULT_SECOND_PARTITION_PATH).getSuccessDeleteFiles() + .size()); + assertTrue(HoodieTestUtils + .doesDataFileExist(basePath, HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH, "000", file1P0C0)); + assertTrue(HoodieTestUtils + .doesDataFileExist(basePath, HoodieTestDataGenerator.DEFAULT_SECOND_PARTITION_PATH, "000", file1P1C0)); // make next commit, with 1 insert & 1 update per partition HoodieTestUtils.createCommitFiles(basePath, "001"); table = HoodieTable.getHoodieTable(new HoodieTableMetaClient(jsc.hadoopConfiguration(), config.getBasePath(), true), config, jsc); - String file2P0C1 = HoodieTestUtils.createNewDataFile(basePath, DEFAULT_FIRST_PARTITION_PATH, "001"); // insert - String file2P1C1 = HoodieTestUtils.createNewDataFile(basePath, DEFAULT_SECOND_PARTITION_PATH, "001"); // insert - HoodieTestUtils.createDataFile(basePath, DEFAULT_FIRST_PARTITION_PATH, "001", file1P0C0); // update - HoodieTestUtils.createDataFile(basePath, DEFAULT_SECOND_PARTITION_PATH, "001", file1P1C0); // update + String file2P0C1 = HoodieTestUtils + .createNewDataFile(basePath, HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH, "001"); // insert + String file2P1C1 = HoodieTestUtils + .createNewDataFile(basePath, HoodieTestDataGenerator.DEFAULT_SECOND_PARTITION_PATH, "001"); // insert + HoodieTestUtils + .createDataFile(basePath, HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH, "001", file1P0C0); // update + HoodieTestUtils + .createDataFile(basePath, HoodieTestDataGenerator.DEFAULT_SECOND_PARTITION_PATH, "001", file1P1C0); // update List hoodieCleanStatsTwo = table.clean(jsc); assertEquals("Must not clean any files", 0, - getCleanStat(hoodieCleanStatsTwo, DEFAULT_FIRST_PARTITION_PATH).getSuccessDeleteFiles().size()); + getCleanStat(hoodieCleanStatsTwo, HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH).getSuccessDeleteFiles() + .size()); assertEquals("Must not clean any files", 0, - getCleanStat(hoodieCleanStatsTwo, DEFAULT_SECOND_PARTITION_PATH).getSuccessDeleteFiles().size()); - assertTrue(HoodieTestUtils.doesDataFileExist(basePath, DEFAULT_FIRST_PARTITION_PATH, "001", file2P0C1)); - assertTrue(HoodieTestUtils.doesDataFileExist(basePath, DEFAULT_SECOND_PARTITION_PATH, "001", file2P1C1)); - assertTrue(HoodieTestUtils.doesDataFileExist(basePath, DEFAULT_FIRST_PARTITION_PATH, "000", file1P0C0)); - assertTrue(HoodieTestUtils.doesDataFileExist(basePath, DEFAULT_SECOND_PARTITION_PATH, "000", file1P1C0)); + getCleanStat(hoodieCleanStatsTwo, HoodieTestDataGenerator.DEFAULT_SECOND_PARTITION_PATH).getSuccessDeleteFiles() + .size()); + assertTrue(HoodieTestUtils + .doesDataFileExist(basePath, HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH, "001", file2P0C1)); + assertTrue(HoodieTestUtils + .doesDataFileExist(basePath, HoodieTestDataGenerator.DEFAULT_SECOND_PARTITION_PATH, "001", file2P1C1)); + assertTrue(HoodieTestUtils + .doesDataFileExist(basePath, HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH, "000", file1P0C0)); + assertTrue(HoodieTestUtils + .doesDataFileExist(basePath, HoodieTestDataGenerator.DEFAULT_SECOND_PARTITION_PATH, "000", file1P1C0)); // make next commit, with 2 updates to existing files, and 1 insert HoodieTestUtils.createCommitFiles(basePath, "002"); table = HoodieTable.getHoodieTable(new HoodieTableMetaClient(jsc.hadoopConfiguration(), config.getBasePath(), true), config, jsc); - HoodieTestUtils.createDataFile(basePath, DEFAULT_FIRST_PARTITION_PATH, "002", file1P0C0); // update - HoodieTestUtils.createDataFile(basePath, DEFAULT_FIRST_PARTITION_PATH, "002", file2P0C1); // update - String file3P0C2 = HoodieTestUtils.createNewDataFile(basePath, DEFAULT_FIRST_PARTITION_PATH, "002"); + HoodieTestUtils + .createDataFile(basePath, HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH, "002", file1P0C0); // update + HoodieTestUtils + .createDataFile(basePath, HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH, "002", file2P0C1); // update + String file3P0C2 = HoodieTestUtils + .createNewDataFile(basePath, HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH, "002"); List hoodieCleanStatsThree = table.clean(jsc); assertEquals("Must not clean any file. We have to keep 1 version before the latest commit time to keep", 0, - getCleanStat(hoodieCleanStatsThree, DEFAULT_FIRST_PARTITION_PATH).getSuccessDeleteFiles().size()); + getCleanStat(hoodieCleanStatsThree, HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH) + .getSuccessDeleteFiles().size()); - assertTrue(HoodieTestUtils.doesDataFileExist(basePath, DEFAULT_FIRST_PARTITION_PATH, "000", file1P0C0)); + assertTrue(HoodieTestUtils + .doesDataFileExist(basePath, HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH, "000", file1P0C0)); // make next commit, with 2 updates to existing files, and 1 insert HoodieTestUtils.createCommitFiles(basePath, "003"); table = HoodieTable.getHoodieTable(new HoodieTableMetaClient(jsc.hadoopConfiguration(), config.getBasePath(), true), config, jsc); - HoodieTestUtils.createDataFile(basePath, DEFAULT_FIRST_PARTITION_PATH, "003", file1P0C0); // update - HoodieTestUtils.createDataFile(basePath, DEFAULT_FIRST_PARTITION_PATH, "003", file2P0C1); // update - String file4P0C3 = HoodieTestUtils.createNewDataFile(basePath, DEFAULT_FIRST_PARTITION_PATH, "003"); + HoodieTestUtils + .createDataFile(basePath, HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH, "003", file1P0C0); // update + HoodieTestUtils + .createDataFile(basePath, HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH, "003", file2P0C1); // update + String file4P0C3 = HoodieTestUtils + .createNewDataFile(basePath, HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH, "003"); List hoodieCleanStatsFour = table.clean(jsc); assertEquals("Must not clean one old file", 1, - getCleanStat(hoodieCleanStatsFour, DEFAULT_FIRST_PARTITION_PATH).getSuccessDeleteFiles().size()); + getCleanStat(hoodieCleanStatsFour, HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH).getSuccessDeleteFiles() + .size()); - assertFalse(HoodieTestUtils.doesDataFileExist(basePath, DEFAULT_FIRST_PARTITION_PATH, "000", file1P0C0)); - assertTrue(HoodieTestUtils.doesDataFileExist(basePath, DEFAULT_FIRST_PARTITION_PATH, "001", file1P0C0)); - assertTrue(HoodieTestUtils.doesDataFileExist(basePath, DEFAULT_FIRST_PARTITION_PATH, "002", file1P0C0)); - assertTrue(HoodieTestUtils.doesDataFileExist(basePath, DEFAULT_FIRST_PARTITION_PATH, "001", file2P0C1)); - assertTrue(HoodieTestUtils.doesDataFileExist(basePath, DEFAULT_FIRST_PARTITION_PATH, "002", file2P0C1)); - assertTrue(HoodieTestUtils.doesDataFileExist(basePath, DEFAULT_FIRST_PARTITION_PATH, "002", file3P0C2)); - assertTrue(HoodieTestUtils.doesDataFileExist(basePath, DEFAULT_FIRST_PARTITION_PATH, "003", file4P0C3)); + assertFalse(HoodieTestUtils + .doesDataFileExist(basePath, HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH, "000", file1P0C0)); + assertTrue(HoodieTestUtils + .doesDataFileExist(basePath, HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH, "001", file1P0C0)); + assertTrue(HoodieTestUtils + .doesDataFileExist(basePath, HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH, "002", file1P0C0)); + assertTrue(HoodieTestUtils + .doesDataFileExist(basePath, HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH, "001", file2P0C1)); + assertTrue(HoodieTestUtils + .doesDataFileExist(basePath, HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH, "002", file2P0C1)); + assertTrue(HoodieTestUtils + .doesDataFileExist(basePath, HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH, "002", file3P0C2)); + assertTrue(HoodieTestUtils + .doesDataFileExist(basePath, HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH, "003", file4P0C3)); // No cleaning on partially written file, with no commit. - HoodieTestUtils.createDataFile(basePath, DEFAULT_FIRST_PARTITION_PATH, "004", file3P0C2); // update + HoodieTestUtils + .createDataFile(basePath, HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH, "004", file3P0C2); // update List hoodieCleanStatsFive = table.clean(jsc); assertEquals("Must not clean any files", 0, - getCleanStat(hoodieCleanStatsFive, DEFAULT_FIRST_PARTITION_PATH).getSuccessDeleteFiles().size()); - assertTrue(HoodieTestUtils.doesDataFileExist(basePath, DEFAULT_FIRST_PARTITION_PATH, "001", file1P0C0)); - assertTrue(HoodieTestUtils.doesDataFileExist(basePath, DEFAULT_FIRST_PARTITION_PATH, "001", file2P0C1)); + getCleanStat(hoodieCleanStatsFive, HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH).getSuccessDeleteFiles() + .size()); + assertTrue(HoodieTestUtils + .doesDataFileExist(basePath, HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH, "001", file1P0C0)); + assertTrue(HoodieTestUtils + .doesDataFileExist(basePath, HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH, "001", file2P0C1)); } /** @@ -697,33 +767,39 @@ public class TestCleaner extends TestHoodieClientBase { // make 1 commit, with 100 files in one partition and 10 in other two HoodieTestUtils.createCommitFiles(basePath, "000"); - List filesP0C0 = createFilesInPartition(DEFAULT_FIRST_PARTITION_PATH, "000", 100); - List filesP1C0 = createFilesInPartition(DEFAULT_SECOND_PARTITION_PATH, "000", 10); - List filesP2C0 = createFilesInPartition(DEFAULT_THIRD_PARTITION_PATH, "000", 10); + List filesP0C0 = createFilesInPartition(HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH, "000", 100); + List filesP1C0 = createFilesInPartition(HoodieTestDataGenerator.DEFAULT_SECOND_PARTITION_PATH, "000", 10); + List filesP2C0 = createFilesInPartition(HoodieTestDataGenerator.DEFAULT_THIRD_PARTITION_PATH, "000", 10); HoodieTestUtils.createCommitFiles(basePath, "001"); - updateAllFilesInPartition(filesP0C0, DEFAULT_FIRST_PARTITION_PATH, "001"); - updateAllFilesInPartition(filesP1C0, DEFAULT_SECOND_PARTITION_PATH, "001"); - updateAllFilesInPartition(filesP2C0, DEFAULT_THIRD_PARTITION_PATH, "001"); + updateAllFilesInPartition(filesP0C0, HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH, "001"); + updateAllFilesInPartition(filesP1C0, HoodieTestDataGenerator.DEFAULT_SECOND_PARTITION_PATH, "001"); + updateAllFilesInPartition(filesP2C0, HoodieTestDataGenerator.DEFAULT_THIRD_PARTITION_PATH, "001"); HoodieTestUtils.createCommitFiles(basePath, "002"); - updateAllFilesInPartition(filesP0C0, DEFAULT_FIRST_PARTITION_PATH, "002"); - updateAllFilesInPartition(filesP1C0, DEFAULT_SECOND_PARTITION_PATH, "002"); - updateAllFilesInPartition(filesP2C0, DEFAULT_THIRD_PARTITION_PATH, "002"); + updateAllFilesInPartition(filesP0C0, HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH, "002"); + updateAllFilesInPartition(filesP1C0, HoodieTestDataGenerator.DEFAULT_SECOND_PARTITION_PATH, "002"); + updateAllFilesInPartition(filesP2C0, HoodieTestDataGenerator.DEFAULT_THIRD_PARTITION_PATH, "002"); HoodieTestUtils.createCommitFiles(basePath, "003"); - updateAllFilesInPartition(filesP0C0, DEFAULT_FIRST_PARTITION_PATH, "003"); - updateAllFilesInPartition(filesP1C0, DEFAULT_SECOND_PARTITION_PATH, "003"); - updateAllFilesInPartition(filesP2C0, DEFAULT_THIRD_PARTITION_PATH, "003"); + updateAllFilesInPartition(filesP0C0, HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH, "003"); + updateAllFilesInPartition(filesP1C0, HoodieTestDataGenerator.DEFAULT_SECOND_PARTITION_PATH, "003"); + updateAllFilesInPartition(filesP2C0, HoodieTestDataGenerator.DEFAULT_THIRD_PARTITION_PATH, "003"); HoodieTable table = HoodieTable.getHoodieTable( new HoodieTableMetaClient(jsc.hadoopConfiguration(), config.getBasePath(), true), config, jsc); List hoodieCleanStats = table.clean(jsc); - assertEquals(100, getCleanStat(hoodieCleanStats, DEFAULT_FIRST_PARTITION_PATH).getSuccessDeleteFiles().size()); - assertEquals(10, getCleanStat(hoodieCleanStats, DEFAULT_SECOND_PARTITION_PATH).getSuccessDeleteFiles().size()); - assertEquals(10, getCleanStat(hoodieCleanStats, DEFAULT_THIRD_PARTITION_PATH).getSuccessDeleteFiles().size()); + assertEquals(100, + getCleanStat(hoodieCleanStats, HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH).getSuccessDeleteFiles() + .size()); + assertEquals(10, + getCleanStat(hoodieCleanStats, HoodieTestDataGenerator.DEFAULT_SECOND_PARTITION_PATH).getSuccessDeleteFiles() + .size()); + assertEquals(10, + getCleanStat(hoodieCleanStats, HoodieTestDataGenerator.DEFAULT_THIRD_PARTITION_PATH).getSuccessDeleteFiles() + .size()); // 3 tasks are expected since the number of partitions is 3 assertEquals(3, stageOneShuffleReadTaskRecordsCountMap.keySet().size()); @@ -780,7 +856,7 @@ public class TestCleaner extends TestHoodieClientBase { /** * Common test method for validating pending compactions * - * @param config Hoodie Write Config + * @param config Hoodie Write Config * @param expNumFilesDeleted Number of files deleted */ public void testPendingCompactions(HoodieWriteConfig config, int expNumFilesDeleted, @@ -806,11 +882,12 @@ public class TestCleaner extends TestHoodieClientBase { {"fileId1", "fileId2", "fileId3", "fileId4", "fileId5", "fileId6", "fileId7"}; int maxNumFileIdsForCompaction = 4; for (int i = 0; i < maxNumFileIds; i++) { - final String fileId = HoodieTestUtils.createDataFile(basePath, DEFAULT_FIRST_PARTITION_PATH, instants[0], - fileIds[i]); - HoodieTestUtils.createNewLogFile(fs, basePath, DEFAULT_FIRST_PARTITION_PATH, instants[0], + final String fileId = HoodieTestUtils + .createDataFile(basePath, HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH, instants[0], + fileIds[i]); + HoodieTestUtils.createNewLogFile(fs, basePath, HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH, instants[0], fileId, Option.empty()); - HoodieTestUtils.createNewLogFile(fs, basePath, DEFAULT_FIRST_PARTITION_PATH, instants[0], + HoodieTestUtils.createNewLogFile(fs, basePath, HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH, instants[0], fileId, Option.of(2)); fileIdToLatestInstantBeforeCompaction.put(fileId, instants[0]); for (int j = 1; j <= i; j++) { @@ -819,7 +896,8 @@ public class TestCleaner extends TestHoodieClientBase { HoodieTable table = HoodieTable.getHoodieTable( new HoodieTableMetaClient(jsc.hadoopConfiguration(), config.getBasePath(), true), config, jsc); - FileSlice slice = table.getRTFileSystemView().getLatestFileSlices(DEFAULT_FIRST_PARTITION_PATH) + FileSlice slice = table.getRTFileSystemView().getLatestFileSlices( + HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH) .filter(fs -> fs.getFileId().equals(fileId)).findFirst().get(); List slices = new ArrayList<>(); if (compactionInstantsToFileSlices.containsKey(compactionInstants[j])) { @@ -828,16 +906,21 @@ public class TestCleaner extends TestHoodieClientBase { slices.add(slice); compactionInstantsToFileSlices.put(compactionInstants[j], slices); // Add log-files to simulate delta-commits after pending compaction - HoodieTestUtils.createNewLogFile(fs, basePath, DEFAULT_FIRST_PARTITION_PATH, compactionInstants[j], + HoodieTestUtils.createNewLogFile(fs, basePath, HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH, + compactionInstants[j], fileId, Option.empty()); - HoodieTestUtils.createNewLogFile(fs, basePath, DEFAULT_FIRST_PARTITION_PATH, compactionInstants[j], + HoodieTestUtils.createNewLogFile(fs, basePath, HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH, + compactionInstants[j], fileId, Option.of(2)); } else { - HoodieTestUtils.createDataFile(basePath, DEFAULT_FIRST_PARTITION_PATH, instants[j], fileId); - HoodieTestUtils.createNewLogFile(fs, basePath, DEFAULT_FIRST_PARTITION_PATH, instants[j], fileId, - Option.empty()); - HoodieTestUtils.createNewLogFile(fs, basePath, DEFAULT_FIRST_PARTITION_PATH, instants[j], fileId, - Option.of(2)); + HoodieTestUtils + .createDataFile(basePath, HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH, instants[j], fileId); + HoodieTestUtils + .createNewLogFile(fs, basePath, HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH, instants[j], fileId, + Option.empty()); + HoodieTestUtils + .createNewLogFile(fs, basePath, HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH, instants[j], fileId, + Option.of(2)); fileIdToLatestInstantBeforeCompaction.put(fileId, instants[j]); } } @@ -848,7 +931,8 @@ public class TestCleaner extends TestHoodieClientBase { List fileSliceList = compactionInstantsToFileSlices.get(instant); if (null != fileSliceList) { HoodieTestUtils.createCompactionRequest(metaClient, instant, - fileSliceList.stream().map(fs -> Pair.of(DEFAULT_FIRST_PARTITION_PATH, fs)).collect(Collectors.toList())); + fileSliceList.stream().map(fs -> Pair.of(HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH, fs)) + .collect(Collectors.toList())); } } @@ -868,8 +952,9 @@ public class TestCleaner extends TestHoodieClientBase { String baseInstantForCompaction = fileIdToLatestInstantBeforeCompaction.get(fileId); Option fileSliceForCompaction = Option.fromJavaOptional( - hoodieTable.getRTFileSystemView().getLatestFileSlicesBeforeOrOn(DEFAULT_FIRST_PARTITION_PATH, - baseInstantForCompaction, true).filter(fs -> fs.getFileId().equals(fileId)).findFirst()); + hoodieTable.getRTFileSystemView().getLatestFileSlicesBeforeOrOn( + HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH, + baseInstantForCompaction, true).filter(fs -> fs.getFileId().equals(fileId)).findFirst()); Assert.assertTrue("Base Instant for Compaction must be preserved", fileSliceForCompaction.isPresent()); Assert.assertTrue("FileSlice has data-file", fileSliceForCompaction.get().getDataFile().isPresent()); Assert.assertEquals("FileSlice has log-files", 2, @@ -885,7 +970,7 @@ public class TestCleaner extends TestHoodieClientBase { Assert.assertTrue("Deleted instant time must be less than pending compaction", HoodieTimeline.compareTimestamps( fileIdToLatestInstantBeforeCompaction.get(fileIdWithCommitTime.getKey()), - fileIdWithCommitTime.getValue(), GREATER)); + fileIdWithCommitTime.getValue(), HoodieTimeline.GREATER)); return true; } return false; diff --git a/hoodie-client/src/test/java/com/uber/hoodie/TestClientRollback.java b/hudi-client/src/test/java/org/apache/hudi/TestClientRollback.java similarity index 93% rename from hoodie-client/src/test/java/com/uber/hoodie/TestClientRollback.java rename to hudi-client/src/test/java/org/apache/hudi/TestClientRollback.java index 8a6f18d20..871e4c38a 100644 --- a/hoodie-client/src/test/java/com/uber/hoodie/TestClientRollback.java +++ b/hudi-client/src/test/java/org/apache/hudi/TestClientRollback.java @@ -16,32 +16,32 @@ * limitations under the License. */ -package com.uber.hoodie; +package org.apache.hudi; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; -import com.uber.hoodie.common.HoodieTestDataGenerator; -import com.uber.hoodie.common.model.HoodieCleaningPolicy; -import com.uber.hoodie.common.model.HoodieDataFile; -import com.uber.hoodie.common.model.HoodieRecord; -import com.uber.hoodie.common.model.HoodieTestUtils; -import com.uber.hoodie.common.table.HoodieTableMetaClient; -import com.uber.hoodie.common.table.TableFileSystemView; -import com.uber.hoodie.common.table.timeline.HoodieInstant; -import com.uber.hoodie.common.util.FSUtils; -import com.uber.hoodie.config.HoodieCompactionConfig; -import com.uber.hoodie.config.HoodieIndexConfig; -import com.uber.hoodie.config.HoodieWriteConfig; -import com.uber.hoodie.exception.HoodieRollbackException; -import com.uber.hoodie.index.HoodieIndex; -import com.uber.hoodie.table.HoodieTable; import java.io.File; import java.io.IOException; import java.util.List; import java.util.stream.Collectors; +import org.apache.hudi.common.HoodieTestDataGenerator; +import org.apache.hudi.common.model.HoodieCleaningPolicy; +import org.apache.hudi.common.model.HoodieDataFile; +import org.apache.hudi.common.model.HoodieRecord; +import org.apache.hudi.common.model.HoodieTestUtils; +import org.apache.hudi.common.table.HoodieTableMetaClient; +import org.apache.hudi.common.table.TableFileSystemView.ReadOptimizedView; +import org.apache.hudi.common.table.timeline.HoodieInstant; +import org.apache.hudi.common.util.FSUtils; +import org.apache.hudi.config.HoodieCompactionConfig; +import org.apache.hudi.config.HoodieIndexConfig; +import org.apache.hudi.config.HoodieWriteConfig; +import org.apache.hudi.exception.HoodieRollbackException; +import org.apache.hudi.index.HoodieIndex; +import org.apache.hudi.table.HoodieTable; import org.apache.spark.api.java.JavaRDD; import org.junit.Test; @@ -105,7 +105,7 @@ public class TestClientRollback extends TestHoodieClientBase { getConfig().shouldAssumeDatePartitioning()); HoodieTableMetaClient metaClient = new HoodieTableMetaClient(jsc.hadoopConfiguration(), basePath); HoodieTable table = HoodieTable.getHoodieTable(metaClient, getConfig(), jsc); - final TableFileSystemView.ReadOptimizedView view1 = table.getROFileSystemView(); + final ReadOptimizedView view1 = table.getROFileSystemView(); List dataFiles = partitionPaths.stream().flatMap(s -> { return view1.getAllDataFiles(s).filter(f -> f.getCommitTime().equals("003")); @@ -130,7 +130,7 @@ public class TestClientRollback extends TestHoodieClientBase { metaClient = new HoodieTableMetaClient(jsc.hadoopConfiguration(), basePath); table = HoodieTable.getHoodieTable(metaClient, getConfig(), jsc); - final TableFileSystemView.ReadOptimizedView view2 = table.getROFileSystemView(); + final ReadOptimizedView view2 = table.getROFileSystemView(); dataFiles = partitionPaths.stream().flatMap(s -> { return view2.getAllDataFiles(s).filter(f -> f.getCommitTime().equals("004")); @@ -151,7 +151,7 @@ public class TestClientRollback extends TestHoodieClientBase { metaClient = new HoodieTableMetaClient(jsc.hadoopConfiguration(), basePath); table = HoodieTable.getHoodieTable(metaClient, getConfig(), jsc); - final TableFileSystemView.ReadOptimizedView view3 = table.getROFileSystemView(); + final ReadOptimizedView view3 = table.getROFileSystemView(); dataFiles = partitionPaths.stream().flatMap(s -> { return view3.getAllDataFiles(s).filter(f -> f.getCommitTime().equals("002")); }).collect(Collectors.toList()); diff --git a/hoodie-client/src/test/java/com/uber/hoodie/TestCompactionAdminClient.java b/hudi-client/src/test/java/org/apache/hudi/TestCompactionAdminClient.java similarity index 95% rename from hoodie-client/src/test/java/com/uber/hoodie/TestCompactionAdminClient.java rename to hudi-client/src/test/java/org/apache/hudi/TestCompactionAdminClient.java index 825d1d14e..f7875a5e8 100644 --- a/hoodie-client/src/test/java/com/uber/hoodie/TestCompactionAdminClient.java +++ b/hudi-client/src/test/java/org/apache/hudi/TestCompactionAdminClient.java @@ -16,23 +16,10 @@ * limitations under the License. */ -package com.uber.hoodie; +package org.apache.hudi; -import static com.uber.hoodie.common.model.HoodieTestUtils.getDefaultHadoopConf; +import static org.apache.hudi.common.model.HoodieTableType.MERGE_ON_READ; -import com.uber.hoodie.CompactionAdminClient.ValidationOpResult; -import com.uber.hoodie.common.model.CompactionOperation; -import com.uber.hoodie.common.model.HoodieLogFile; -import com.uber.hoodie.common.model.HoodieTableType; -import com.uber.hoodie.common.model.HoodieTestUtils; -import com.uber.hoodie.common.table.HoodieTableMetaClient; -import com.uber.hoodie.common.table.view.HoodieTableFileSystemView; -import com.uber.hoodie.common.util.CompactionTestUtils; -import com.uber.hoodie.common.util.CompactionUtils; -import com.uber.hoodie.common.util.Option; -import com.uber.hoodie.common.util.collection.Pair; -import com.uber.hoodie.exception.HoodieException; -import com.uber.hoodie.exception.HoodieIOException; import java.io.IOException; import java.util.Arrays; import java.util.HashSet; @@ -40,6 +27,18 @@ import java.util.List; import java.util.Map; import java.util.Set; import java.util.stream.Collectors; +import org.apache.hudi.CompactionAdminClient.ValidationOpResult; +import org.apache.hudi.common.model.CompactionOperation; +import org.apache.hudi.common.model.HoodieLogFile; +import org.apache.hudi.common.model.HoodieTestUtils; +import org.apache.hudi.common.table.HoodieTableMetaClient; +import org.apache.hudi.common.table.view.HoodieTableFileSystemView; +import org.apache.hudi.common.util.CompactionTestUtils; +import org.apache.hudi.common.util.CompactionUtils; +import org.apache.hudi.common.util.Option; +import org.apache.hudi.common.util.collection.Pair; +import org.apache.hudi.exception.HoodieException; +import org.apache.hudi.exception.HoodieIOException; import org.junit.Assert; import org.junit.Before; import org.junit.Test; @@ -52,7 +51,7 @@ public class TestCompactionAdminClient extends TestHoodieClientBase { @Before public void init() throws IOException { super.init(); - metaClient = HoodieTestUtils.initTableType(getDefaultHadoopConf(), basePath, HoodieTableType.MERGE_ON_READ); + metaClient = HoodieTestUtils.initTableType(HoodieTestUtils.getDefaultHadoopConf(), basePath, MERGE_ON_READ); client = new CompactionAdminClient(jsc, basePath); } diff --git a/hoodie-client/src/test/java/com/uber/hoodie/TestConsistencyGuard.java b/hudi-client/src/test/java/org/apache/hudi/TestConsistencyGuard.java similarity index 95% rename from hoodie-client/src/test/java/com/uber/hoodie/TestConsistencyGuard.java rename to hudi-client/src/test/java/org/apache/hudi/TestConsistencyGuard.java index 7712c1c78..161708f8e 100644 --- a/hoodie-client/src/test/java/com/uber/hoodie/TestConsistencyGuard.java +++ b/hudi-client/src/test/java/org/apache/hudi/TestConsistencyGuard.java @@ -16,13 +16,8 @@ * limitations under the License. */ -package com.uber.hoodie; +package org.apache.hudi; -import com.uber.hoodie.common.HoodieClientTestUtils; -import com.uber.hoodie.common.util.ConsistencyGuard; -import com.uber.hoodie.common.util.ConsistencyGuardConfig; -import com.uber.hoodie.common.util.FSUtils; -import com.uber.hoodie.common.util.FailSafeConsistencyGuard; import java.io.IOException; import java.util.Arrays; import java.util.concurrent.TimeoutException; @@ -30,6 +25,11 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.LocalFileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hudi.common.HoodieClientTestUtils; +import org.apache.hudi.common.util.ConsistencyGuard; +import org.apache.hudi.common.util.ConsistencyGuardConfig; +import org.apache.hudi.common.util.FSUtils; +import org.apache.hudi.common.util.FailSafeConsistencyGuard; import org.junit.Before; import org.junit.Test; import org.junit.rules.TemporaryFolder; diff --git a/hoodie-client/src/test/java/com/uber/hoodie/TestHoodieClientBase.java b/hudi-client/src/test/java/org/apache/hudi/TestHoodieClientBase.java similarity index 92% rename from hoodie-client/src/test/java/com/uber/hoodie/TestHoodieClientBase.java rename to hudi-client/src/test/java/org/apache/hudi/TestHoodieClientBase.java index 5d20d9e53..f826c838e 100644 --- a/hoodie-client/src/test/java/com/uber/hoodie/TestHoodieClientBase.java +++ b/hudi-client/src/test/java/org/apache/hudi/TestHoodieClientBase.java @@ -16,37 +16,12 @@ * limitations under the License. */ -package com.uber.hoodie; +package org.apache.hudi; -import static com.uber.hoodie.common.HoodieTestDataGenerator.DEFAULT_PARTITION_DEPTH; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; -import com.uber.hoodie.common.HoodieCleanStat; -import com.uber.hoodie.common.HoodieClientTestUtils; -import com.uber.hoodie.common.HoodieTestDataGenerator; -import com.uber.hoodie.common.TestRawTripPayload.MetadataMergeWriteStatus; -import com.uber.hoodie.common.model.HoodiePartitionMetadata; -import com.uber.hoodie.common.model.HoodieRecord; -import com.uber.hoodie.common.model.HoodieTableType; -import com.uber.hoodie.common.model.HoodieTestUtils; -import com.uber.hoodie.common.table.HoodieTableMetaClient; -import com.uber.hoodie.common.table.HoodieTimeline; -import com.uber.hoodie.common.table.SyncableFileSystemView; -import com.uber.hoodie.common.table.timeline.HoodieActiveTimeline; -import com.uber.hoodie.common.table.view.FileSystemViewStorageConfig; -import com.uber.hoodie.common.table.view.FileSystemViewStorageType; -import com.uber.hoodie.common.util.ConsistencyGuardConfig; -import com.uber.hoodie.common.util.FSUtils; -import com.uber.hoodie.common.util.Option; -import com.uber.hoodie.config.HoodieCompactionConfig; -import com.uber.hoodie.config.HoodieIndexConfig; -import com.uber.hoodie.config.HoodieStorageConfig; -import com.uber.hoodie.config.HoodieWriteConfig; -import com.uber.hoodie.index.HoodieIndex; -import com.uber.hoodie.index.HoodieIndex.IndexType; -import com.uber.hoodie.table.HoodieTable; import java.io.File; import java.io.IOException; import java.io.Serializable; @@ -59,12 +34,37 @@ import java.util.Set; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.LocalFileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hudi.common.HoodieCleanStat; +import org.apache.hudi.common.HoodieClientTestUtils; +import org.apache.hudi.common.HoodieTestDataGenerator; +import org.apache.hudi.common.TestRawTripPayload.MetadataMergeWriteStatus; +import org.apache.hudi.common.model.HoodiePartitionMetadata; +import org.apache.hudi.common.model.HoodieRecord; +import org.apache.hudi.common.model.HoodieTableType; +import org.apache.hudi.common.model.HoodieTestUtils; +import org.apache.hudi.common.table.HoodieTableMetaClient; +import org.apache.hudi.common.table.HoodieTimeline; +import org.apache.hudi.common.table.SyncableFileSystemView; +import org.apache.hudi.common.table.timeline.HoodieActiveTimeline; +import org.apache.hudi.common.table.view.FileSystemViewStorageConfig; +import org.apache.hudi.common.table.view.FileSystemViewStorageType; +import org.apache.hudi.common.util.ConsistencyGuardConfig; +import org.apache.hudi.common.util.FSUtils; +import org.apache.hudi.common.util.Option; +import org.apache.hudi.config.HoodieCompactionConfig; +import org.apache.hudi.config.HoodieIndexConfig; +import org.apache.hudi.config.HoodieStorageConfig; +import org.apache.hudi.config.HoodieWriteConfig; +import org.apache.hudi.index.HoodieIndex; +import org.apache.hudi.index.HoodieIndex.IndexType; +import org.apache.hudi.table.HoodieTable; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; import org.apache.spark.api.java.JavaRDD; import org.apache.spark.api.java.JavaSparkContext; import org.apache.spark.sql.SQLContext; import org.junit.After; +import org.junit.Assert; import org.junit.Before; import org.junit.rules.TemporaryFolder; @@ -234,7 +234,7 @@ public class TestHoodieClientBase implements Serializable { assertTrue(HoodiePartitionMetadata.hasPartitionMetadata(fs, new Path(basePath, partitionPath))); HoodiePartitionMetadata pmeta = new HoodiePartitionMetadata(fs, new Path(basePath, partitionPath)); pmeta.readFromFS(); - assertEquals(DEFAULT_PARTITION_DEPTH, pmeta.getPartitionDepth()); + Assert.assertEquals(HoodieTestDataGenerator.DEFAULT_PARTITION_DEPTH, pmeta.getPartitionDepth()); } } @@ -440,7 +440,7 @@ public class TestHoodieClientBase implements Serializable { if (assertForCommit) { assertEquals("Expecting " + expTotalCommits + " commits.", expTotalCommits, timeline.findInstantsAfter(initCommitTime, Integer.MAX_VALUE).countInstants()); - assertEquals("Latest commit should be " + newCommitTime, newCommitTime, + Assert.assertEquals("Latest commit should be " + newCommitTime, newCommitTime, timeline.lastInstant().get().getTimestamp()); assertEquals("Must contain " + expRecordsInThisCommit + " records", expRecordsInThisCommit, HoodieClientTestUtils.readCommit(basePath, sqlContext, timeline, newCommitTime).count()); diff --git a/hoodie-client/src/test/java/com/uber/hoodie/TestHoodieClientOnCopyOnWriteStorage.java b/hudi-client/src/test/java/org/apache/hudi/TestHoodieClientOnCopyOnWriteStorage.java similarity index 93% rename from hoodie-client/src/test/java/com/uber/hoodie/TestHoodieClientOnCopyOnWriteStorage.java rename to hudi-client/src/test/java/org/apache/hudi/TestHoodieClientOnCopyOnWriteStorage.java index caf96289a..5ed2d38df 100644 --- a/hoodie-client/src/test/java/com/uber/hoodie/TestHoodieClientOnCopyOnWriteStorage.java +++ b/hudi-client/src/test/java/org/apache/hudi/TestHoodieClientOnCopyOnWriteStorage.java @@ -16,9 +16,9 @@ * limitations under the License. */ -package com.uber.hoodie; +package org.apache.hudi; -import static com.uber.hoodie.common.table.HoodieTableMetaClient.MARKER_EXTN; +import static org.apache.hudi.common.util.ParquetUtils.readRowKeysFromParquet; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; @@ -26,31 +26,6 @@ import static org.junit.Assert.fail; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; -import com.uber.hoodie.common.HoodieClientTestUtils; -import com.uber.hoodie.common.HoodieTestDataGenerator; -import com.uber.hoodie.common.model.HoodieCommitMetadata; -import com.uber.hoodie.common.model.HoodieDataFile; -import com.uber.hoodie.common.model.HoodieKey; -import com.uber.hoodie.common.model.HoodieRecord; -import com.uber.hoodie.common.model.HoodieRollingStat; -import com.uber.hoodie.common.model.HoodieRollingStatMetadata; -import com.uber.hoodie.common.model.HoodieTestUtils; -import com.uber.hoodie.common.table.HoodieTableMetaClient; -import com.uber.hoodie.common.table.HoodieTimeline; -import com.uber.hoodie.common.table.TableFileSystemView; -import com.uber.hoodie.common.table.timeline.HoodieInstant; -import com.uber.hoodie.common.util.ConsistencyGuardConfig; -import com.uber.hoodie.common.util.FSUtils; -import com.uber.hoodie.common.util.Option; -import com.uber.hoodie.common.util.ParquetUtils; -import com.uber.hoodie.common.util.collection.Pair; -import com.uber.hoodie.config.HoodieCompactionConfig; -import com.uber.hoodie.config.HoodieStorageConfig; -import com.uber.hoodie.config.HoodieWriteConfig; -import com.uber.hoodie.exception.HoodieCommitException; -import com.uber.hoodie.exception.HoodieIOException; -import com.uber.hoodie.index.HoodieIndex; -import com.uber.hoodie.table.HoodieTable; import java.io.FileInputStream; import java.io.IOException; import java.util.ArrayList; @@ -65,6 +40,31 @@ import java.util.stream.Collectors; import org.apache.avro.generic.GenericRecord; import org.apache.commons.io.IOUtils; import org.apache.hadoop.fs.Path; +import org.apache.hudi.common.HoodieClientTestUtils; +import org.apache.hudi.common.HoodieTestDataGenerator; +import org.apache.hudi.common.model.HoodieCommitMetadata; +import org.apache.hudi.common.model.HoodieDataFile; +import org.apache.hudi.common.model.HoodieKey; +import org.apache.hudi.common.model.HoodieRecord; +import org.apache.hudi.common.model.HoodieRollingStat; +import org.apache.hudi.common.model.HoodieRollingStatMetadata; +import org.apache.hudi.common.model.HoodieTestUtils; +import org.apache.hudi.common.table.HoodieTableMetaClient; +import org.apache.hudi.common.table.HoodieTimeline; +import org.apache.hudi.common.table.TableFileSystemView.ReadOptimizedView; +import org.apache.hudi.common.table.timeline.HoodieInstant; +import org.apache.hudi.common.util.ConsistencyGuardConfig; +import org.apache.hudi.common.util.FSUtils; +import org.apache.hudi.common.util.Option; +import org.apache.hudi.common.util.ParquetUtils; +import org.apache.hudi.common.util.collection.Pair; +import org.apache.hudi.config.HoodieCompactionConfig; +import org.apache.hudi.config.HoodieStorageConfig; +import org.apache.hudi.config.HoodieWriteConfig; +import org.apache.hudi.exception.HoodieCommitException; +import org.apache.hudi.exception.HoodieIOException; +import org.apache.hudi.index.HoodieIndex; +import org.apache.hudi.table.HoodieTable; import org.apache.spark.api.java.JavaRDD; import org.junit.Assert; import org.junit.Test; @@ -362,7 +362,7 @@ public class TestHoodieClientOnCopyOnWriteStorage extends TestHoodieClientBase { assertEquals("Just 1 file needs to be added.", 1, statuses.size()); String file1 = statuses.get(0).getFileId(); - assertEquals("file should contain 100 records", ParquetUtils.readRowKeysFromParquet(jsc.hadoopConfiguration(), + Assert.assertEquals("file should contain 100 records", readRowKeysFromParquet(jsc.hadoopConfiguration(), new Path(basePath, statuses.get(0).getStat().getPath())).size(), 100); // Update + Inserts such that they just expand file1 @@ -383,7 +383,7 @@ public class TestHoodieClientOnCopyOnWriteStorage extends TestHoodieClientBase { assertEquals("Existing file should be expanded", commitTime1, statuses.get(0).getStat().getPrevCommit()); Path newFile = new Path(basePath, statuses.get(0).getStat().getPath()); assertEquals("file should contain 140 records", - ParquetUtils.readRowKeysFromParquet(jsc.hadoopConfiguration(), newFile).size(), 140); + readRowKeysFromParquet(jsc.hadoopConfiguration(), newFile).size(), 140); List records = ParquetUtils.readAvroRecords(jsc.hadoopConfiguration(), newFile); for (GenericRecord record : records) { @@ -408,7 +408,7 @@ public class TestHoodieClientOnCopyOnWriteStorage extends TestHoodieClientBase { HoodieTableMetaClient metadata = new HoodieTableMetaClient(jsc.hadoopConfiguration(), basePath); HoodieTable table = getHoodieTable(metadata, config); - TableFileSystemView.ReadOptimizedView fileSystemView = table.getROFileSystemView(); + ReadOptimizedView fileSystemView = table.getROFileSystemView(); List files = fileSystemView.getLatestDataFilesBeforeOrOn(testPartitionPath, commitTime3) .collect(Collectors.toList()); int numTotalInsertsInCommit3 = 0; @@ -472,7 +472,7 @@ public class TestHoodieClientOnCopyOnWriteStorage extends TestHoodieClientBase { assertEquals("Just 1 file needs to be added.", 1, statuses.size()); String file1 = statuses.get(0).getFileId(); - assertEquals("file should contain 100 records", ParquetUtils.readRowKeysFromParquet(jsc.hadoopConfiguration(), + assertEquals("file should contain 100 records", readRowKeysFromParquet(jsc.hadoopConfiguration(), new Path(basePath, statuses.get(0).getStat().getPath())).size(), 100); // Second, set of Inserts should just expand file1 @@ -489,7 +489,7 @@ public class TestHoodieClientOnCopyOnWriteStorage extends TestHoodieClientBase { assertEquals("Existing file should be expanded", commitTime1, statuses.get(0).getStat().getPrevCommit()); Path newFile = new Path(basePath, statuses.get(0).getStat().getPath()); assertEquals("file should contain 140 records", - ParquetUtils.readRowKeysFromParquet(jsc.hadoopConfiguration(), newFile).size(), 140); + readRowKeysFromParquet(jsc.hadoopConfiguration(), newFile).size(), 140); List records = ParquetUtils.readAvroRecords(jsc.hadoopConfiguration(), newFile); for (GenericRecord record : records) { @@ -704,7 +704,7 @@ public class TestHoodieClientOnCopyOnWriteStorage extends TestHoodieClientBase { // This should fail the commit String partitionPath = Arrays.stream(fs.globStatus(new Path(String.format("%s/*/*/*/*", metaClient.getMarkerFolderPath(commitTime))), - path -> path.toString().endsWith(MARKER_EXTN))).limit(1) + path -> path.toString().endsWith(HoodieTableMetaClient.MARKER_EXTN))).limit(1) .map(status -> status.getPath().getParent().toString()).collect(Collectors.toList()).get(0); Path markerFilePath = new Path(String.format("%s/%s", partitionPath, FSUtils.makeMarkerFile(commitTime, "1-0-1", UUID.randomUUID().toString()))); diff --git a/hoodie-client/src/test/java/com/uber/hoodie/TestHoodieReadClient.java b/hudi-client/src/test/java/org/apache/hudi/TestHoodieReadClient.java similarity index 97% rename from hoodie-client/src/test/java/com/uber/hoodie/TestHoodieReadClient.java rename to hudi-client/src/test/java/org/apache/hudi/TestHoodieReadClient.java index 4bd85db42..b66184e10 100644 --- a/hoodie-client/src/test/java/com/uber/hoodie/TestHoodieReadClient.java +++ b/hudi-client/src/test/java/org/apache/hudi/TestHoodieReadClient.java @@ -16,18 +16,18 @@ * limitations under the License. */ -package com.uber.hoodie; +package org.apache.hudi; import static org.junit.Assert.assertTrue; -import com.uber.hoodie.common.model.HoodieRecord; -import com.uber.hoodie.common.util.Option; -import com.uber.hoodie.config.HoodieWriteConfig; import java.io.IOException; import java.util.Arrays; import java.util.Collection; import java.util.List; import java.util.stream.Collectors; +import org.apache.hudi.common.model.HoodieRecord; +import org.apache.hudi.common.util.Option; +import org.apache.hudi.config.HoodieWriteConfig; import org.apache.spark.api.java.JavaRDD; import org.junit.Assert; import org.junit.Test; diff --git a/hoodie-client/src/test/java/com/uber/hoodie/TestMultiFS.java b/hudi-client/src/test/java/org/apache/hudi/TestMultiFS.java similarity index 89% rename from hoodie-client/src/test/java/com/uber/hoodie/TestMultiFS.java rename to hudi-client/src/test/java/org/apache/hudi/TestMultiFS.java index 881481b18..ba94e12d7 100644 --- a/hoodie-client/src/test/java/com/uber/hoodie/TestMultiFS.java +++ b/hudi-client/src/test/java/org/apache/hudi/TestMultiFS.java @@ -16,30 +16,30 @@ * limitations under the License. */ -package com.uber.hoodie; +package org.apache.hudi; import static org.junit.Assert.assertEquals; -import com.uber.hoodie.common.HoodieClientTestUtils; -import com.uber.hoodie.common.HoodieTestDataGenerator; -import com.uber.hoodie.common.minicluster.HdfsTestService; -import com.uber.hoodie.common.model.HoodieAvroPayload; -import com.uber.hoodie.common.model.HoodieRecord; -import com.uber.hoodie.common.model.HoodieTableType; -import com.uber.hoodie.common.model.HoodieTestUtils; -import com.uber.hoodie.common.table.HoodieTableMetaClient; -import com.uber.hoodie.common.table.HoodieTimeline; -import com.uber.hoodie.common.table.timeline.HoodieActiveTimeline; -import com.uber.hoodie.common.util.FSUtils; -import com.uber.hoodie.config.HoodieIndexConfig; -import com.uber.hoodie.config.HoodieWriteConfig; -import com.uber.hoodie.index.HoodieIndex; import java.io.Serializable; import java.util.List; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.hudi.common.HoodieClientTestUtils; +import org.apache.hudi.common.HoodieTestDataGenerator; +import org.apache.hudi.common.minicluster.HdfsTestService; +import org.apache.hudi.common.model.HoodieAvroPayload; +import org.apache.hudi.common.model.HoodieRecord; +import org.apache.hudi.common.model.HoodieTableType; +import org.apache.hudi.common.model.HoodieTestUtils; +import org.apache.hudi.common.table.HoodieTableMetaClient; +import org.apache.hudi.common.table.HoodieTimeline; +import org.apache.hudi.common.table.timeline.HoodieActiveTimeline; +import org.apache.hudi.common.util.FSUtils; +import org.apache.hudi.config.HoodieIndexConfig; +import org.apache.hudi.config.HoodieWriteConfig; +import org.apache.hudi.index.HoodieIndex; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; import org.apache.spark.SparkConf; diff --git a/hoodie-client/src/test/java/com/uber/hoodie/TestWriteStatus.java b/hudi-client/src/test/java/org/apache/hudi/TestWriteStatus.java similarity index 96% rename from hoodie-client/src/test/java/com/uber/hoodie/TestWriteStatus.java rename to hudi-client/src/test/java/org/apache/hudi/TestWriteStatus.java index 9b16e33b4..352acd500 100644 --- a/hoodie-client/src/test/java/com/uber/hoodie/TestWriteStatus.java +++ b/hudi-client/src/test/java/org/apache/hudi/TestWriteStatus.java @@ -16,13 +16,13 @@ * limitations under the License. */ -package com.uber.hoodie; +package org.apache.hudi; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; -import com.uber.hoodie.common.model.HoodieRecord; import java.io.IOException; +import org.apache.hudi.common.model.HoodieRecord; import org.junit.Test; import org.mockito.Mockito; diff --git a/hoodie-client/src/test/java/com/uber/hoodie/common/HoodieClientTestUtils.java b/hudi-client/src/test/java/org/apache/hudi/common/HoodieClientTestUtils.java similarity index 90% rename from hoodie-client/src/test/java/com/uber/hoodie/common/HoodieClientTestUtils.java rename to hudi-client/src/test/java/org/apache/hudi/common/HoodieClientTestUtils.java index 501691f0b..ba930aa3d 100644 --- a/hoodie-client/src/test/java/com/uber/hoodie/common/HoodieClientTestUtils.java +++ b/hudi-client/src/test/java/org/apache/hudi/common/HoodieClientTestUtils.java @@ -16,26 +16,8 @@ * limitations under the License. */ -package com.uber.hoodie.common; +package org.apache.hudi.common; -import com.uber.hoodie.HoodieReadClient; -import com.uber.hoodie.WriteStatus; -import com.uber.hoodie.avro.HoodieAvroWriteSupport; -import com.uber.hoodie.common.model.HoodieCommitMetadata; -import com.uber.hoodie.common.model.HoodieDataFile; -import com.uber.hoodie.common.model.HoodieRecord; -import com.uber.hoodie.common.model.HoodieTestUtils; -import com.uber.hoodie.common.table.HoodieTableMetaClient; -import com.uber.hoodie.common.table.HoodieTimeline; -import com.uber.hoodie.common.table.TableFileSystemView; -import com.uber.hoodie.common.table.timeline.HoodieInstant; -import com.uber.hoodie.common.table.view.HoodieTableFileSystemView; -import com.uber.hoodie.common.util.FSUtils; -import com.uber.hoodie.common.util.HoodieAvroUtils; -import com.uber.hoodie.config.HoodieStorageConfig; -import com.uber.hoodie.exception.HoodieException; -import com.uber.hoodie.io.storage.HoodieParquetConfig; -import com.uber.hoodie.io.storage.HoodieParquetWriter; import java.io.File; import java.io.IOException; import java.io.RandomAccessFile; @@ -52,6 +34,24 @@ import org.apache.avro.Schema; import org.apache.avro.generic.GenericRecord; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hudi.HoodieReadClient; +import org.apache.hudi.WriteStatus; +import org.apache.hudi.avro.HoodieAvroWriteSupport; +import org.apache.hudi.common.model.HoodieCommitMetadata; +import org.apache.hudi.common.model.HoodieDataFile; +import org.apache.hudi.common.model.HoodieRecord; +import org.apache.hudi.common.model.HoodieTestUtils; +import org.apache.hudi.common.table.HoodieTableMetaClient; +import org.apache.hudi.common.table.HoodieTimeline; +import org.apache.hudi.common.table.TableFileSystemView.ReadOptimizedView; +import org.apache.hudi.common.table.timeline.HoodieInstant; +import org.apache.hudi.common.table.view.HoodieTableFileSystemView; +import org.apache.hudi.common.util.FSUtils; +import org.apache.hudi.common.util.HoodieAvroUtils; +import org.apache.hudi.config.HoodieStorageConfig; +import org.apache.hudi.exception.HoodieException; +import org.apache.hudi.io.storage.HoodieParquetConfig; +import org.apache.hudi.io.storage.HoodieParquetWriter; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; import org.apache.parquet.avro.AvroSchemaConverter; @@ -179,7 +179,7 @@ public class HoodieClientTestUtils { try { HoodieTableMetaClient metaClient = new HoodieTableMetaClient(fs.getConf(), basePath, true); for (String path : paths) { - TableFileSystemView.ReadOptimizedView fileSystemView = new HoodieTableFileSystemView( + ReadOptimizedView fileSystemView = new HoodieTableFileSystemView( metaClient, metaClient.getCommitsTimeline().filterCompletedInstants(), fs.globStatus(new Path(path))); List latestFiles = fileSystemView.getLatestDataFiles().collect(Collectors.toList()); for (HoodieDataFile file : latestFiles) { diff --git a/hoodie-client/src/test/java/com/uber/hoodie/common/HoodieMergeOnReadTestUtils.java b/hudi-client/src/test/java/org/apache/hudi/common/HoodieMergeOnReadTestUtils.java similarity index 93% rename from hoodie-client/src/test/java/com/uber/hoodie/common/HoodieMergeOnReadTestUtils.java rename to hudi-client/src/test/java/org/apache/hudi/common/HoodieMergeOnReadTestUtils.java index b122fdc88..5d03f3af4 100644 --- a/hoodie-client/src/test/java/com/uber/hoodie/common/HoodieMergeOnReadTestUtils.java +++ b/hudi-client/src/test/java/org/apache/hudi/common/HoodieMergeOnReadTestUtils.java @@ -16,13 +16,8 @@ * limitations under the License. */ -package com.uber.hoodie.common; +package org.apache.hudi.common; -import static com.uber.hoodie.common.HoodieTestDataGenerator.TRIP_EXAMPLE_SCHEMA; - -import com.uber.hoodie.common.model.HoodieTestUtils; -import com.uber.hoodie.common.util.HoodieAvroUtils; -import com.uber.hoodie.hadoop.realtime.HoodieRealtimeInputFormat; import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; @@ -38,6 +33,9 @@ import org.apache.hadoop.io.Writable; import org.apache.hadoop.mapred.InputSplit; import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.mapred.RecordReader; +import org.apache.hudi.common.model.HoodieTestUtils; +import org.apache.hudi.common.util.HoodieAvroUtils; +import org.apache.hudi.hadoop.realtime.HoodieRealtimeInputFormat; /** * Utility methods to aid in testing MergeOnRead (workaround for HoodieReadClient for MOR) @@ -47,7 +45,7 @@ public class HoodieMergeOnReadTestUtils { public static List getRecordsUsingInputFormat(List inputPaths, String basePath) throws IOException { JobConf jobConf = new JobConf(); - Schema schema = HoodieAvroUtils.addMetadataFields(Schema.parse(TRIP_EXAMPLE_SCHEMA)); + Schema schema = HoodieAvroUtils.addMetadataFields(Schema.parse(HoodieTestDataGenerator.TRIP_EXAMPLE_SCHEMA)); HoodieRealtimeInputFormat inputFormat = new HoodieRealtimeInputFormat(); setPropsForInputFormat(inputFormat, jobConf, schema, basePath); return inputPaths.stream().map(path -> { diff --git a/hoodie-client/src/test/java/com/uber/hoodie/common/HoodieTestDataGenerator.java b/hudi-client/src/test/java/org/apache/hudi/common/HoodieTestDataGenerator.java similarity index 94% rename from hoodie-client/src/test/java/com/uber/hoodie/common/HoodieTestDataGenerator.java rename to hudi-client/src/test/java/org/apache/hudi/common/HoodieTestDataGenerator.java index 5c7a028f0..bc90b680c 100644 --- a/hoodie-client/src/test/java/com/uber/hoodie/common/HoodieTestDataGenerator.java +++ b/hudi-client/src/test/java/org/apache/hudi/common/HoodieTestDataGenerator.java @@ -16,23 +16,8 @@ * limitations under the License. */ -package com.uber.hoodie.common; +package org.apache.hudi.common; -import com.uber.hoodie.avro.model.HoodieCompactionPlan; -import com.uber.hoodie.common.model.HoodieAvroPayload; -import com.uber.hoodie.common.model.HoodieCommitMetadata; -import com.uber.hoodie.common.model.HoodieKey; -import com.uber.hoodie.common.model.HoodiePartitionMetadata; -import com.uber.hoodie.common.model.HoodieRecord; -import com.uber.hoodie.common.model.HoodieTestUtils; -import com.uber.hoodie.common.table.HoodieTableMetaClient; -import com.uber.hoodie.common.table.HoodieTimeline; -import com.uber.hoodie.common.table.timeline.HoodieInstant; -import com.uber.hoodie.common.util.AvroUtils; -import com.uber.hoodie.common.util.FSUtils; -import com.uber.hoodie.common.util.HoodieAvroUtils; -import com.uber.hoodie.common.util.Option; -import com.uber.hoodie.exception.HoodieIOException; import java.io.IOException; import java.io.Serializable; import java.nio.charset.StandardCharsets; @@ -55,6 +40,21 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hudi.avro.model.HoodieCompactionPlan; +import org.apache.hudi.common.model.HoodieAvroPayload; +import org.apache.hudi.common.model.HoodieCommitMetadata; +import org.apache.hudi.common.model.HoodieKey; +import org.apache.hudi.common.model.HoodiePartitionMetadata; +import org.apache.hudi.common.model.HoodieRecord; +import org.apache.hudi.common.model.HoodieTestUtils; +import org.apache.hudi.common.table.HoodieTableMetaClient; +import org.apache.hudi.common.table.HoodieTimeline; +import org.apache.hudi.common.table.timeline.HoodieInstant; +import org.apache.hudi.common.util.AvroUtils; +import org.apache.hudi.common.util.FSUtils; +import org.apache.hudi.common.util.HoodieAvroUtils; +import org.apache.hudi.common.util.Option; +import org.apache.hudi.exception.HoodieIOException; /** * Class to be used in tests to keep generating test inserts and updates against a corpus. diff --git a/hoodie-client/src/test/java/com/uber/hoodie/common/TestRawTripPayload.java b/hudi-client/src/test/java/org/apache/hudi/common/TestRawTripPayload.java similarity index 96% rename from hoodie-client/src/test/java/com/uber/hoodie/common/TestRawTripPayload.java rename to hudi-client/src/test/java/org/apache/hudi/common/TestRawTripPayload.java index 6fcb5a877..32297cb37 100644 --- a/hoodie-client/src/test/java/com/uber/hoodie/common/TestRawTripPayload.java +++ b/hudi-client/src/test/java/org/apache/hudi/common/TestRawTripPayload.java @@ -16,14 +16,9 @@ * limitations under the License. */ -package com.uber.hoodie.common; +package org.apache.hudi.common; import com.fasterxml.jackson.databind.ObjectMapper; -import com.uber.hoodie.WriteStatus; -import com.uber.hoodie.avro.MercifulJsonConverter; -import com.uber.hoodie.common.model.HoodieRecord; -import com.uber.hoodie.common.model.HoodieRecordPayload; -import com.uber.hoodie.common.util.Option; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; @@ -38,6 +33,11 @@ import java.util.zip.InflaterInputStream; import org.apache.avro.Schema; import org.apache.avro.generic.IndexedRecord; import org.apache.commons.io.IOUtils; +import org.apache.hudi.WriteStatus; +import org.apache.hudi.avro.MercifulJsonConverter; +import org.apache.hudi.common.model.HoodieRecord; +import org.apache.hudi.common.model.HoodieRecordPayload; +import org.apache.hudi.common.util.Option; /** * Example row change event based on some example data used by testcases. The data avro schema is diff --git a/hoodie-client/src/test/java/com/uber/hoodie/config/HoodieWriteConfigTest.java b/hudi-client/src/test/java/org/apache/hudi/config/HoodieWriteConfigTest.java similarity index 96% rename from hoodie-client/src/test/java/com/uber/hoodie/config/HoodieWriteConfigTest.java rename to hudi-client/src/test/java/org/apache/hudi/config/HoodieWriteConfigTest.java index 5bdcce727..82a774b6a 100644 --- a/hoodie-client/src/test/java/com/uber/hoodie/config/HoodieWriteConfigTest.java +++ b/hudi-client/src/test/java/org/apache/hudi/config/HoodieWriteConfigTest.java @@ -16,18 +16,18 @@ * limitations under the License. */ -package com.uber.hoodie.config; +package org.apache.hudi.config; import static org.junit.Assert.assertEquals; import com.google.common.collect.Maps; -import com.uber.hoodie.config.HoodieWriteConfig.Builder; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.util.Date; import java.util.Map; import java.util.Properties; +import org.apache.hudi.config.HoodieWriteConfig.Builder; import org.junit.Test; public class HoodieWriteConfigTest { diff --git a/hoodie-client/src/test/java/com/uber/hoodie/func/TestBoundedInMemoryExecutor.java b/hudi-client/src/test/java/org/apache/hudi/func/TestBoundedInMemoryExecutor.java similarity index 85% rename from hoodie-client/src/test/java/com/uber/hoodie/func/TestBoundedInMemoryExecutor.java rename to hudi-client/src/test/java/org/apache/hudi/func/TestBoundedInMemoryExecutor.java index 1e10fc0d8..e0485efd9 100644 --- a/hoodie-client/src/test/java/com/uber/hoodie/func/TestBoundedInMemoryExecutor.java +++ b/hudi-client/src/test/java/org/apache/hudi/func/TestBoundedInMemoryExecutor.java @@ -16,21 +16,21 @@ * limitations under the License. */ -package com.uber.hoodie.func; +package org.apache.hudi.func; -import static com.uber.hoodie.func.CopyOnWriteLazyInsertIterable.getTransformFunction; +import static org.apache.hudi.func.CopyOnWriteLazyInsertIterable.getTransformFunction; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; -import com.uber.hoodie.common.HoodieTestDataGenerator; -import com.uber.hoodie.common.model.HoodieRecord; -import com.uber.hoodie.common.table.timeline.HoodieActiveTimeline; -import com.uber.hoodie.common.util.Option; -import com.uber.hoodie.common.util.queue.BoundedInMemoryQueueConsumer; -import com.uber.hoodie.config.HoodieWriteConfig; -import com.uber.hoodie.func.CopyOnWriteLazyInsertIterable.HoodieInsertValueGenResult; import java.util.List; import org.apache.avro.generic.IndexedRecord; +import org.apache.hudi.common.HoodieTestDataGenerator; +import org.apache.hudi.common.model.HoodieRecord; +import org.apache.hudi.common.table.timeline.HoodieActiveTimeline; +import org.apache.hudi.common.util.Option; +import org.apache.hudi.common.util.queue.BoundedInMemoryQueueConsumer; +import org.apache.hudi.config.HoodieWriteConfig; +import org.apache.hudi.func.CopyOnWriteLazyInsertIterable.HoodieInsertValueGenResult; import org.junit.After; import org.junit.Assert; import org.junit.Test; diff --git a/hoodie-client/src/test/java/com/uber/hoodie/func/TestBoundedInMemoryQueue.java b/hudi-client/src/test/java/org/apache/hudi/func/TestBoundedInMemoryQueue.java similarity index 94% rename from hoodie-client/src/test/java/com/uber/hoodie/func/TestBoundedInMemoryQueue.java rename to hudi-client/src/test/java/org/apache/hudi/func/TestBoundedInMemoryQueue.java index 6b760a8e7..043cc5ec0 100644 --- a/hoodie-client/src/test/java/com/uber/hoodie/func/TestBoundedInMemoryQueue.java +++ b/hudi-client/src/test/java/org/apache/hudi/func/TestBoundedInMemoryQueue.java @@ -16,24 +16,12 @@ * limitations under the License. */ -package com.uber.hoodie.func; +package org.apache.hudi.func; -import static com.uber.hoodie.func.CopyOnWriteLazyInsertIterable.getTransformFunction; +import static org.apache.hudi.func.CopyOnWriteLazyInsertIterable.getTransformFunction; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; -import com.uber.hoodie.common.HoodieTestDataGenerator; -import com.uber.hoodie.common.model.HoodieRecord; -import com.uber.hoodie.common.table.timeline.HoodieActiveTimeline; -import com.uber.hoodie.common.util.DefaultSizeEstimator; -import com.uber.hoodie.common.util.Option; -import com.uber.hoodie.common.util.SizeEstimator; -import com.uber.hoodie.common.util.queue.BoundedInMemoryQueue; -import com.uber.hoodie.common.util.queue.BoundedInMemoryQueueProducer; -import com.uber.hoodie.common.util.queue.FunctionBasedQueueProducer; -import com.uber.hoodie.common.util.queue.IteratorBasedQueueProducer; -import com.uber.hoodie.exception.HoodieException; -import com.uber.hoodie.func.CopyOnWriteLazyInsertIterable.HoodieInsertValueGenResult; import java.util.ArrayList; import java.util.HashMap; import java.util.Iterator; @@ -49,6 +37,18 @@ import java.util.stream.Collectors; import java.util.stream.IntStream; import org.apache.avro.generic.IndexedRecord; import org.apache.commons.io.FileUtils; +import org.apache.hudi.common.HoodieTestDataGenerator; +import org.apache.hudi.common.model.HoodieRecord; +import org.apache.hudi.common.table.timeline.HoodieActiveTimeline; +import org.apache.hudi.common.util.DefaultSizeEstimator; +import org.apache.hudi.common.util.Option; +import org.apache.hudi.common.util.SizeEstimator; +import org.apache.hudi.common.util.queue.BoundedInMemoryQueue; +import org.apache.hudi.common.util.queue.BoundedInMemoryQueueProducer; +import org.apache.hudi.common.util.queue.FunctionBasedQueueProducer; +import org.apache.hudi.common.util.queue.IteratorBasedQueueProducer; +import org.apache.hudi.exception.HoodieException; +import org.apache.hudi.func.CopyOnWriteLazyInsertIterable.HoodieInsertValueGenResult; import org.junit.After; import org.junit.Assert; import org.junit.Before; diff --git a/hoodie-client/src/test/java/com/uber/hoodie/func/TestParquetReaderIterator.java b/hudi-client/src/test/java/org/apache/hudi/func/TestParquetReaderIterator.java similarity index 96% rename from hoodie-client/src/test/java/com/uber/hoodie/func/TestParquetReaderIterator.java rename to hudi-client/src/test/java/org/apache/hudi/func/TestParquetReaderIterator.java index 944b9e499..ee17d1e71 100644 --- a/hoodie-client/src/test/java/com/uber/hoodie/func/TestParquetReaderIterator.java +++ b/hudi-client/src/test/java/org/apache/hudi/func/TestParquetReaderIterator.java @@ -16,13 +16,13 @@ * limitations under the License. */ -package com.uber.hoodie.func; +package org.apache.hudi.func; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; -import com.uber.hoodie.exception.HoodieIOException; import java.io.IOException; +import org.apache.hudi.exception.HoodieIOException; import org.apache.parquet.hadoop.ParquetReader; import org.junit.Assert; import org.junit.Test; diff --git a/hoodie-client/src/test/java/com/uber/hoodie/func/TestUpdateMapFunction.java b/hudi-client/src/test/java/org/apache/hudi/func/TestUpdateMapFunction.java similarity index 89% rename from hoodie-client/src/test/java/com/uber/hoodie/func/TestUpdateMapFunction.java rename to hudi-client/src/test/java/org/apache/hudi/func/TestUpdateMapFunction.java index f6cd9659f..bfa1a49e7 100644 --- a/hoodie-client/src/test/java/com/uber/hoodie/func/TestUpdateMapFunction.java +++ b/hudi-client/src/test/java/org/apache/hudi/func/TestUpdateMapFunction.java @@ -16,24 +16,10 @@ * limitations under the License. */ -package com.uber.hoodie.func; +package org.apache.hudi.func; import static org.junit.Assert.fail; -import com.uber.hoodie.WriteStatus; -import com.uber.hoodie.common.HoodieClientTestUtils; -import com.uber.hoodie.common.TestRawTripPayload; -import com.uber.hoodie.common.model.HoodieKey; -import com.uber.hoodie.common.model.HoodieRecord; -import com.uber.hoodie.common.model.HoodieRecordLocation; -import com.uber.hoodie.common.model.HoodieTestUtils; -import com.uber.hoodie.common.table.HoodieTimeline; -import com.uber.hoodie.common.util.FSUtils; -import com.uber.hoodie.common.util.ParquetUtils; -import com.uber.hoodie.config.HoodieWriteConfig; -import com.uber.hoodie.io.HoodieCreateHandle; -import com.uber.hoodie.io.HoodieMergeHandle; -import com.uber.hoodie.table.HoodieCopyOnWriteTable; import java.io.File; import java.io.Serializable; import java.util.ArrayList; @@ -44,6 +30,20 @@ import org.apache.avro.generic.GenericRecord; import org.apache.commons.io.IOUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; +import org.apache.hudi.WriteStatus; +import org.apache.hudi.common.HoodieClientTestUtils; +import org.apache.hudi.common.TestRawTripPayload; +import org.apache.hudi.common.model.HoodieKey; +import org.apache.hudi.common.model.HoodieRecord; +import org.apache.hudi.common.model.HoodieRecordLocation; +import org.apache.hudi.common.model.HoodieTestUtils; +import org.apache.hudi.common.table.HoodieTimeline; +import org.apache.hudi.common.util.FSUtils; +import org.apache.hudi.common.util.ParquetUtils; +import org.apache.hudi.config.HoodieWriteConfig; +import org.apache.hudi.io.HoodieCreateHandle; +import org.apache.hudi.io.HoodieMergeHandle; +import org.apache.hudi.table.HoodieCopyOnWriteTable; import org.apache.parquet.avro.AvroReadSupport; import org.apache.spark.api.java.JavaSparkContext; import org.junit.After; diff --git a/hoodie-client/src/test/java/com/uber/hoodie/index/TestHBaseQPSResourceAllocator.java b/hudi-client/src/test/java/org/apache/hudi/index/TestHBaseQPSResourceAllocator.java similarity index 89% rename from hoodie-client/src/test/java/com/uber/hoodie/index/TestHBaseQPSResourceAllocator.java rename to hudi-client/src/test/java/org/apache/hudi/index/TestHBaseQPSResourceAllocator.java index 9eca394aa..696159ecb 100644 --- a/hoodie-client/src/test/java/com/uber/hoodie/index/TestHBaseQPSResourceAllocator.java +++ b/hudi-client/src/test/java/org/apache/hudi/index/TestHBaseQPSResourceAllocator.java @@ -16,23 +16,23 @@ * limitations under the License. */ -package com.uber.hoodie.index; +package org.apache.hudi.index; -import com.uber.hoodie.common.HoodieClientTestUtils; -import com.uber.hoodie.common.HoodieTestDataGenerator; -import com.uber.hoodie.common.model.HoodieTestUtils; -import com.uber.hoodie.common.util.Option; -import com.uber.hoodie.config.HoodieCompactionConfig; -import com.uber.hoodie.config.HoodieHBaseIndexConfig; -import com.uber.hoodie.config.HoodieIndexConfig; -import com.uber.hoodie.config.HoodieStorageConfig; -import com.uber.hoodie.config.HoodieWriteConfig; -import com.uber.hoodie.index.hbase.DefaultHBaseQPSResourceAllocator; -import com.uber.hoodie.index.hbase.HBaseIndex; -import com.uber.hoodie.index.hbase.HBaseIndexQPSResourceAllocator; import java.io.File; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hudi.common.HoodieClientTestUtils; +import org.apache.hudi.common.HoodieTestDataGenerator; +import org.apache.hudi.common.model.HoodieTestUtils; +import org.apache.hudi.common.util.Option; +import org.apache.hudi.config.HoodieCompactionConfig; +import org.apache.hudi.config.HoodieHBaseIndexConfig; +import org.apache.hudi.config.HoodieIndexConfig; +import org.apache.hudi.config.HoodieStorageConfig; +import org.apache.hudi.config.HoodieWriteConfig; +import org.apache.hudi.index.hbase.DefaultHBaseQPSResourceAllocator; +import org.apache.hudi.index.hbase.HBaseIndex; +import org.apache.hudi.index.hbase.HBaseIndexQPSResourceAllocator; import org.apache.spark.api.java.JavaSparkContext; import org.junit.After; import org.junit.AfterClass; diff --git a/hoodie-client/src/test/java/com/uber/hoodie/index/TestHbaseIndex.java b/hudi-client/src/test/java/org/apache/hudi/index/TestHbaseIndex.java similarity index 95% rename from hoodie-client/src/test/java/com/uber/hoodie/index/TestHbaseIndex.java rename to hudi-client/src/test/java/org/apache/hudi/index/TestHbaseIndex.java index 5ff72208c..7ae01c7f4 100644 --- a/hoodie-client/src/test/java/com/uber/hoodie/index/TestHbaseIndex.java +++ b/hudi-client/src/test/java/org/apache/hudi/index/TestHbaseIndex.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package com.uber.hoodie.index; +package org.apache.hudi.index; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; @@ -24,24 +24,6 @@ import static org.mockito.Matchers.anyObject; import static org.mockito.Mockito.atMost; import static org.mockito.Mockito.times; -import com.uber.hoodie.HoodieWriteClient; -import com.uber.hoodie.WriteStatus; -import com.uber.hoodie.common.HoodieClientTestUtils; -import com.uber.hoodie.common.HoodieTestDataGenerator; -import com.uber.hoodie.common.model.HoodieRecord; -import com.uber.hoodie.common.model.HoodieTestUtils; -import com.uber.hoodie.common.model.HoodieWriteStat; -import com.uber.hoodie.common.table.HoodieTableMetaClient; -import com.uber.hoodie.config.HoodieCompactionConfig; -import com.uber.hoodie.config.HoodieHBaseIndexConfig; -import com.uber.hoodie.config.HoodieIndexConfig; -import com.uber.hoodie.config.HoodieStorageConfig; -import com.uber.hoodie.config.HoodieWriteConfig; -import com.uber.hoodie.index.hbase.DefaultHBaseQPSResourceAllocator; -import com.uber.hoodie.index.hbase.HBaseIndex; -import com.uber.hoodie.index.hbase.HBaseIndex.HbasePutBatchSizeCalculator; -import com.uber.hoodie.index.hbase.HBaseIndexQPSResourceAllocator; -import com.uber.hoodie.table.HoodieTable; import java.io.File; import java.util.Arrays; import java.util.List; @@ -55,6 +37,24 @@ import org.apache.hadoop.hbase.client.HTable; import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hudi.HoodieWriteClient; +import org.apache.hudi.WriteStatus; +import org.apache.hudi.common.HoodieClientTestUtils; +import org.apache.hudi.common.HoodieTestDataGenerator; +import org.apache.hudi.common.model.HoodieRecord; +import org.apache.hudi.common.model.HoodieTestUtils; +import org.apache.hudi.common.model.HoodieWriteStat; +import org.apache.hudi.common.table.HoodieTableMetaClient; +import org.apache.hudi.config.HoodieCompactionConfig; +import org.apache.hudi.config.HoodieHBaseIndexConfig; +import org.apache.hudi.config.HoodieIndexConfig; +import org.apache.hudi.config.HoodieStorageConfig; +import org.apache.hudi.config.HoodieWriteConfig; +import org.apache.hudi.index.hbase.DefaultHBaseQPSResourceAllocator; +import org.apache.hudi.index.hbase.HBaseIndex; +import org.apache.hudi.index.hbase.HBaseIndex.HbasePutBatchSizeCalculator; +import org.apache.hudi.index.hbase.HBaseIndexQPSResourceAllocator; +import org.apache.hudi.table.HoodieTable; import org.apache.spark.api.java.JavaRDD; import org.apache.spark.api.java.JavaSparkContext; import org.junit.After; @@ -67,7 +67,6 @@ import org.junit.Test; import org.junit.rules.TemporaryFolder; import org.junit.runners.MethodSorters; import org.mockito.Mockito; - import scala.Tuple2; /** diff --git a/hoodie-client/src/test/java/com/uber/hoodie/index/TestHoodieIndex.java b/hudi-client/src/test/java/org/apache/hudi/index/TestHoodieIndex.java similarity index 87% rename from hoodie-client/src/test/java/com/uber/hoodie/index/TestHoodieIndex.java rename to hudi-client/src/test/java/org/apache/hudi/index/TestHoodieIndex.java index cfc30a73b..e692fa756 100644 --- a/hoodie-client/src/test/java/com/uber/hoodie/index/TestHoodieIndex.java +++ b/hudi-client/src/test/java/org/apache/hudi/index/TestHoodieIndex.java @@ -16,19 +16,19 @@ * limitations under the License. */ -package com.uber.hoodie.index; +package org.apache.hudi.index; import static org.junit.Assert.assertTrue; -import com.uber.hoodie.common.HoodieClientTestUtils; -import com.uber.hoodie.common.model.HoodieTestUtils; -import com.uber.hoodie.config.HoodieHBaseIndexConfig; -import com.uber.hoodie.config.HoodieIndexConfig; -import com.uber.hoodie.config.HoodieWriteConfig; -import com.uber.hoodie.index.bloom.HoodieBloomIndex; -import com.uber.hoodie.index.hbase.HBaseIndex; import java.io.File; import java.io.IOException; +import org.apache.hudi.common.HoodieClientTestUtils; +import org.apache.hudi.common.model.HoodieTestUtils; +import org.apache.hudi.config.HoodieHBaseIndexConfig; +import org.apache.hudi.config.HoodieIndexConfig; +import org.apache.hudi.config.HoodieWriteConfig; +import org.apache.hudi.index.bloom.HoodieBloomIndex; +import org.apache.hudi.index.hbase.HBaseIndex; import org.apache.spark.api.java.JavaSparkContext; import org.junit.After; import org.junit.Before; diff --git a/hoodie-client/src/test/java/com/uber/hoodie/index/bloom/TestBucketizedBloomCheckPartitioner.java b/hudi-client/src/test/java/org/apache/hudi/index/bloom/TestBucketizedBloomCheckPartitioner.java similarity index 97% rename from hoodie-client/src/test/java/com/uber/hoodie/index/bloom/TestBucketizedBloomCheckPartitioner.java rename to hudi-client/src/test/java/org/apache/hudi/index/bloom/TestBucketizedBloomCheckPartitioner.java index db2567615..e2282b0e9 100644 --- a/hoodie-client/src/test/java/com/uber/hoodie/index/bloom/TestBucketizedBloomCheckPartitioner.java +++ b/hudi-client/src/test/java/org/apache/hudi/index/bloom/TestBucketizedBloomCheckPartitioner.java @@ -16,17 +16,17 @@ * limitations under the License. */ -package com.uber.hoodie.index.bloom; +package org.apache.hudi.index.bloom; import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; -import com.uber.hoodie.common.util.collection.Pair; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.stream.Collectors; import java.util.stream.IntStream; +import org.apache.hudi.common.util.collection.Pair; import org.junit.Test; public class TestBucketizedBloomCheckPartitioner { diff --git a/hoodie-client/src/test/java/com/uber/hoodie/index/bloom/TestHoodieBloomIndex.java b/hudi-client/src/test/java/org/apache/hudi/index/bloom/TestHoodieBloomIndex.java similarity index 96% rename from hoodie-client/src/test/java/com/uber/hoodie/index/bloom/TestHoodieBloomIndex.java rename to hudi-client/src/test/java/org/apache/hudi/index/bloom/TestHoodieBloomIndex.java index 15512a804..dffe3da41 100644 --- a/hoodie-client/src/test/java/com/uber/hoodie/index/bloom/TestHoodieBloomIndex.java +++ b/hudi-client/src/test/java/org/apache/hudi/index/bloom/TestHoodieBloomIndex.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package com.uber.hoodie.index.bloom; +package org.apache.hudi.index.bloom; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -26,21 +26,6 @@ import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import com.google.common.collect.Lists; -import com.uber.hoodie.common.BloomFilter; -import com.uber.hoodie.common.HoodieClientTestUtils; -import com.uber.hoodie.common.TestRawTripPayload; -import com.uber.hoodie.common.model.HoodieKey; -import com.uber.hoodie.common.model.HoodieRecord; -import com.uber.hoodie.common.model.HoodieTestUtils; -import com.uber.hoodie.common.table.HoodieTableMetaClient; -import com.uber.hoodie.common.util.FSUtils; -import com.uber.hoodie.common.util.HoodieAvroUtils; -import com.uber.hoodie.common.util.Option; -import com.uber.hoodie.common.util.collection.Pair; -import com.uber.hoodie.config.HoodieIndexConfig; -import com.uber.hoodie.config.HoodieWriteConfig; -import com.uber.hoodie.io.HoodieKeyLookupHandle; -import com.uber.hoodie.table.HoodieTable; import java.io.File; import java.io.IOException; import java.util.Arrays; @@ -55,6 +40,21 @@ import org.apache.avro.Schema; import org.apache.commons.io.IOUtils; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hudi.common.BloomFilter; +import org.apache.hudi.common.HoodieClientTestUtils; +import org.apache.hudi.common.TestRawTripPayload; +import org.apache.hudi.common.model.HoodieKey; +import org.apache.hudi.common.model.HoodieRecord; +import org.apache.hudi.common.model.HoodieTestUtils; +import org.apache.hudi.common.table.HoodieTableMetaClient; +import org.apache.hudi.common.util.FSUtils; +import org.apache.hudi.common.util.HoodieAvroUtils; +import org.apache.hudi.common.util.Option; +import org.apache.hudi.common.util.collection.Pair; +import org.apache.hudi.config.HoodieIndexConfig; +import org.apache.hudi.config.HoodieWriteConfig; +import org.apache.hudi.io.HoodieKeyLookupHandle; +import org.apache.hudi.table.HoodieTable; import org.apache.spark.api.java.JavaPairRDD; import org.apache.spark.api.java.JavaRDD; import org.apache.spark.api.java.JavaSparkContext; diff --git a/hoodie-client/src/test/java/com/uber/hoodie/index/bloom/TestHoodieGlobalBloomIndex.java b/hudi-client/src/test/java/org/apache/hudi/index/bloom/TestHoodieGlobalBloomIndex.java similarity index 96% rename from hoodie-client/src/test/java/com/uber/hoodie/index/bloom/TestHoodieGlobalBloomIndex.java rename to hudi-client/src/test/java/org/apache/hudi/index/bloom/TestHoodieGlobalBloomIndex.java index 4048a5e3f..90c2e0cf7 100644 --- a/hoodie-client/src/test/java/com/uber/hoodie/index/bloom/TestHoodieGlobalBloomIndex.java +++ b/hudi-client/src/test/java/org/apache/hudi/index/bloom/TestHoodieGlobalBloomIndex.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package com.uber.hoodie.index.bloom; +package org.apache.hudi.index.bloom; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -25,17 +25,6 @@ import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; import com.google.common.collect.Lists; -import com.uber.hoodie.common.HoodieClientTestUtils; -import com.uber.hoodie.common.TestRawTripPayload; -import com.uber.hoodie.common.model.HoodieKey; -import com.uber.hoodie.common.model.HoodiePartitionMetadata; -import com.uber.hoodie.common.model.HoodieRecord; -import com.uber.hoodie.common.model.HoodieTestUtils; -import com.uber.hoodie.common.table.HoodieTableMetaClient; -import com.uber.hoodie.common.util.FSUtils; -import com.uber.hoodie.common.util.HoodieAvroUtils; -import com.uber.hoodie.config.HoodieWriteConfig; -import com.uber.hoodie.table.HoodieTable; import java.io.File; import java.io.IOException; import java.util.Arrays; @@ -47,6 +36,17 @@ import java.util.stream.Collectors; import org.apache.avro.Schema; import org.apache.commons.io.IOUtils; import org.apache.hadoop.fs.FileSystem; +import org.apache.hudi.common.HoodieClientTestUtils; +import org.apache.hudi.common.TestRawTripPayload; +import org.apache.hudi.common.model.HoodieKey; +import org.apache.hudi.common.model.HoodiePartitionMetadata; +import org.apache.hudi.common.model.HoodieRecord; +import org.apache.hudi.common.model.HoodieTestUtils; +import org.apache.hudi.common.table.HoodieTableMetaClient; +import org.apache.hudi.common.util.FSUtils; +import org.apache.hudi.common.util.HoodieAvroUtils; +import org.apache.hudi.config.HoodieWriteConfig; +import org.apache.hudi.table.HoodieTable; import org.apache.spark.api.java.JavaPairRDD; import org.apache.spark.api.java.JavaRDD; import org.apache.spark.api.java.JavaSparkContext; diff --git a/hoodie-client/src/test/java/com/uber/hoodie/index/bloom/TestKeyRangeLookupTree.java b/hudi-client/src/test/java/org/apache/hudi/index/bloom/TestKeyRangeLookupTree.java similarity index 99% rename from hoodie-client/src/test/java/com/uber/hoodie/index/bloom/TestKeyRangeLookupTree.java rename to hudi-client/src/test/java/org/apache/hudi/index/bloom/TestKeyRangeLookupTree.java index d914a1723..ba7400398 100644 --- a/hoodie-client/src/test/java/com/uber/hoodie/index/bloom/TestKeyRangeLookupTree.java +++ b/hudi-client/src/test/java/org/apache/hudi/index/bloom/TestKeyRangeLookupTree.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package com.uber.hoodie.index.bloom; +package org.apache.hudi.index.bloom; import static junit.framework.TestCase.assertEquals; import static junit.framework.TestCase.assertTrue; diff --git a/hoodie-client/src/test/java/com/uber/hoodie/io/TestHoodieCommitArchiveLog.java b/hudi-client/src/test/java/org/apache/hudi/io/TestHoodieCommitArchiveLog.java similarity index 81% rename from hoodie-client/src/test/java/com/uber/hoodie/io/TestHoodieCommitArchiveLog.java rename to hudi-client/src/test/java/org/apache/hudi/io/TestHoodieCommitArchiveLog.java index 9819ddfff..29f2bb7f3 100644 --- a/hoodie-client/src/test/java/com/uber/hoodie/io/TestHoodieCommitArchiveLog.java +++ b/hudi-client/src/test/java/org/apache/hudi/io/TestHoodieCommitArchiveLog.java @@ -16,29 +16,13 @@ * limitations under the License. */ -package com.uber.hoodie.io; +package org.apache.hudi.io; -import static com.uber.hoodie.common.table.HoodieTimeline.COMPACTION_ACTION; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; import com.google.common.collect.Sets; -import com.uber.hoodie.avro.model.HoodieArchivedMetaEntry; -import com.uber.hoodie.common.HoodieClientTestUtils; -import com.uber.hoodie.common.HoodieTestDataGenerator; -import com.uber.hoodie.common.minicluster.HdfsTestService; -import com.uber.hoodie.common.model.HoodieLogFile; -import com.uber.hoodie.common.model.HoodieTestUtils; -import com.uber.hoodie.common.table.HoodieTableMetaClient; -import com.uber.hoodie.common.table.HoodieTimeline; -import com.uber.hoodie.common.table.log.HoodieLogFormat; -import com.uber.hoodie.common.table.log.block.HoodieAvroDataBlock; -import com.uber.hoodie.common.table.timeline.HoodieActiveTimeline; -import com.uber.hoodie.common.table.timeline.HoodieInstant; -import com.uber.hoodie.common.table.timeline.HoodieInstant.State; -import com.uber.hoodie.config.HoodieCompactionConfig; -import com.uber.hoodie.config.HoodieWriteConfig; import java.io.File; import java.io.IOException; import java.util.ArrayList; @@ -52,6 +36,22 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.hudi.avro.model.HoodieArchivedMetaEntry; +import org.apache.hudi.common.HoodieClientTestUtils; +import org.apache.hudi.common.HoodieTestDataGenerator; +import org.apache.hudi.common.minicluster.HdfsTestService; +import org.apache.hudi.common.model.HoodieLogFile; +import org.apache.hudi.common.model.HoodieTestUtils; +import org.apache.hudi.common.table.HoodieTableMetaClient; +import org.apache.hudi.common.table.HoodieTimeline; +import org.apache.hudi.common.table.log.HoodieLogFormat; +import org.apache.hudi.common.table.log.HoodieLogFormat.Reader; +import org.apache.hudi.common.table.log.block.HoodieAvroDataBlock; +import org.apache.hudi.common.table.timeline.HoodieActiveTimeline; +import org.apache.hudi.common.table.timeline.HoodieInstant; +import org.apache.hudi.common.table.timeline.HoodieInstant.State; +import org.apache.hudi.config.HoodieCompactionConfig; +import org.apache.hudi.config.HoodieWriteConfig; import org.apache.spark.api.java.JavaSparkContext; import org.junit.After; import org.junit.AfterClass; @@ -142,45 +142,45 @@ public class TestHoodieCommitArchiveLog { HoodieTestUtils.init(hadoopConf, basePath); // Requested Compaction HoodieTestDataGenerator.createCompactionAuxiliaryMetadata(basePath, - new HoodieInstant(State.REQUESTED, COMPACTION_ACTION, "100"), dfs.getConf()); + new HoodieInstant(State.REQUESTED, HoodieTimeline.COMPACTION_ACTION, "100"), dfs.getConf()); // Inflight Compaction HoodieTestDataGenerator.createCompactionAuxiliaryMetadata(basePath, - new HoodieInstant(State.INFLIGHT, COMPACTION_ACTION, "100"), dfs.getConf()); + new HoodieInstant(State.INFLIGHT, HoodieTimeline.COMPACTION_ACTION, "100"), dfs.getConf()); HoodieTestDataGenerator.createCommitFile(basePath, "100", dfs.getConf()); // Requested Compaction HoodieTestDataGenerator.createCompactionAuxiliaryMetadata(basePath, - new HoodieInstant(State.REQUESTED, COMPACTION_ACTION, "101"), dfs.getConf()); + new HoodieInstant(State.REQUESTED, HoodieTimeline.COMPACTION_ACTION, "101"), dfs.getConf()); // Inflight Compaction HoodieTestDataGenerator.createCompactionAuxiliaryMetadata(basePath, - new HoodieInstant(State.INFLIGHT, COMPACTION_ACTION, "101"), dfs.getConf()); + new HoodieInstant(State.INFLIGHT, HoodieTimeline.COMPACTION_ACTION, "101"), dfs.getConf()); HoodieTestDataGenerator.createCommitFile(basePath, "101", dfs.getConf()); // Requested Compaction HoodieTestDataGenerator.createCompactionAuxiliaryMetadata(basePath, - new HoodieInstant(State.REQUESTED, COMPACTION_ACTION, "102"), dfs.getConf()); + new HoodieInstant(State.REQUESTED, HoodieTimeline.COMPACTION_ACTION, "102"), dfs.getConf()); // Inflight Compaction HoodieTestDataGenerator.createCompactionAuxiliaryMetadata(basePath, - new HoodieInstant(State.INFLIGHT, COMPACTION_ACTION, "102"), dfs.getConf()); + new HoodieInstant(State.INFLIGHT, HoodieTimeline.COMPACTION_ACTION, "102"), dfs.getConf()); HoodieTestDataGenerator.createCommitFile(basePath, "102", dfs.getConf()); // Requested Compaction HoodieTestDataGenerator.createCompactionAuxiliaryMetadata(basePath, - new HoodieInstant(State.REQUESTED, COMPACTION_ACTION, "103"), dfs.getConf()); + new HoodieInstant(State.REQUESTED, HoodieTimeline.COMPACTION_ACTION, "103"), dfs.getConf()); // Inflight Compaction HoodieTestDataGenerator.createCompactionAuxiliaryMetadata(basePath, - new HoodieInstant(State.INFLIGHT, COMPACTION_ACTION, "103"), dfs.getConf()); + new HoodieInstant(State.INFLIGHT, HoodieTimeline.COMPACTION_ACTION, "103"), dfs.getConf()); HoodieTestDataGenerator.createCommitFile(basePath, "103", dfs.getConf()); // Requested Compaction HoodieTestDataGenerator.createCompactionAuxiliaryMetadata(basePath, - new HoodieInstant(State.REQUESTED, COMPACTION_ACTION, "104"), dfs.getConf()); + new HoodieInstant(State.REQUESTED, HoodieTimeline.COMPACTION_ACTION, "104"), dfs.getConf()); // Inflight Compaction HoodieTestDataGenerator.createCompactionAuxiliaryMetadata(basePath, - new HoodieInstant(State.INFLIGHT, COMPACTION_ACTION, "104"), dfs.getConf()); + new HoodieInstant(State.INFLIGHT, HoodieTimeline.COMPACTION_ACTION, "104"), dfs.getConf()); HoodieTestDataGenerator.createCommitFile(basePath, "104", dfs.getConf()); // Requested Compaction HoodieTestDataGenerator.createCompactionAuxiliaryMetadata(basePath, - new HoodieInstant(State.REQUESTED, COMPACTION_ACTION, "105"), dfs.getConf()); + new HoodieInstant(State.REQUESTED, HoodieTimeline.COMPACTION_ACTION, "105"), dfs.getConf()); // Inflight Compaction HoodieTestDataGenerator.createCompactionAuxiliaryMetadata(basePath, - new HoodieInstant(State.INFLIGHT, COMPACTION_ACTION, "105"), dfs.getConf()); + new HoodieInstant(State.INFLIGHT, HoodieTimeline.COMPACTION_ACTION, "105"), dfs.getConf()); HoodieTestDataGenerator.createCommitFile(basePath, "105", dfs.getConf()); HoodieTableMetaClient metaClient = new HoodieTableMetaClient(dfs.getConf(), basePath); @@ -222,32 +222,32 @@ public class TestHoodieCommitArchiveLog { HoodieActiveTimeline.VALID_EXTENSIONS_IN_ACTIVE_TIMELINE); assertEquals("Should delete all compaction instants < 104", 4, instants.size()); assertFalse("Requested Compaction must be absent for 100", instants.contains( - new HoodieInstant(State.REQUESTED, COMPACTION_ACTION, "100"))); + new HoodieInstant(State.REQUESTED, HoodieTimeline.COMPACTION_ACTION, "100"))); assertFalse("Inflight Compaction must be absent for 100", instants.contains( - new HoodieInstant(State.INFLIGHT, COMPACTION_ACTION, "100"))); + new HoodieInstant(State.INFLIGHT, HoodieTimeline.COMPACTION_ACTION, "100"))); assertFalse("Requested Compaction must be absent for 101", instants.contains( - new HoodieInstant(State.REQUESTED, COMPACTION_ACTION, "101"))); + new HoodieInstant(State.REQUESTED, HoodieTimeline.COMPACTION_ACTION, "101"))); assertFalse("Inflight Compaction must be absent for 101", instants.contains( - new HoodieInstant(State.INFLIGHT, COMPACTION_ACTION, "101"))); + new HoodieInstant(State.INFLIGHT, HoodieTimeline.COMPACTION_ACTION, "101"))); assertFalse("Requested Compaction must be absent for 102", instants.contains( - new HoodieInstant(State.REQUESTED, COMPACTION_ACTION, "102"))); + new HoodieInstant(State.REQUESTED, HoodieTimeline.COMPACTION_ACTION, "102"))); assertFalse("Inflight Compaction must be absent for 102", instants.contains( - new HoodieInstant(State.INFLIGHT, COMPACTION_ACTION, "102"))); + new HoodieInstant(State.INFLIGHT, HoodieTimeline.COMPACTION_ACTION, "102"))); assertFalse("Requested Compaction must be absent for 103", instants.contains( - new HoodieInstant(State.REQUESTED, COMPACTION_ACTION, "103"))); + new HoodieInstant(State.REQUESTED, HoodieTimeline.COMPACTION_ACTION, "103"))); assertFalse("Inflight Compaction must be absent for 103", instants.contains( - new HoodieInstant(State.INFLIGHT, COMPACTION_ACTION, "103"))); + new HoodieInstant(State.INFLIGHT, HoodieTimeline.COMPACTION_ACTION, "103"))); assertTrue("Requested Compaction must be present for 104", instants.contains( - new HoodieInstant(State.REQUESTED, COMPACTION_ACTION, "104"))); + new HoodieInstant(State.REQUESTED, HoodieTimeline.COMPACTION_ACTION, "104"))); assertTrue("Inflight Compaction must be present for 104", instants.contains( - new HoodieInstant(State.INFLIGHT, COMPACTION_ACTION, "104"))); + new HoodieInstant(State.INFLIGHT, HoodieTimeline.COMPACTION_ACTION, "104"))); assertTrue("Requested Compaction must be present for 105", instants.contains( - new HoodieInstant(State.REQUESTED, COMPACTION_ACTION, "105"))); + new HoodieInstant(State.REQUESTED, HoodieTimeline.COMPACTION_ACTION, "105"))); assertTrue("Inflight Compaction must be present for 105", instants.contains( - new HoodieInstant(State.INFLIGHT, COMPACTION_ACTION, "105"))); + new HoodieInstant(State.INFLIGHT, HoodieTimeline.COMPACTION_ACTION, "105"))); //read the file - HoodieLogFormat.Reader reader = HoodieLogFormat.newReader(dfs, + Reader reader = HoodieLogFormat.newReader(dfs, new HoodieLogFile(new Path(basePath + "/.hoodie/.commits_.archive.1_1-0-1")), HoodieArchivedMetaEntry.getClassSchema()); @@ -287,31 +287,31 @@ public class TestHoodieCommitArchiveLog { HoodieCommitArchiveLog archiveLog = new HoodieCommitArchiveLog(cfg, metaClient); // Requested Compaction HoodieTestDataGenerator.createCompactionAuxiliaryMetadata(basePath, - new HoodieInstant(State.REQUESTED, COMPACTION_ACTION, "100"), dfs.getConf()); + new HoodieInstant(State.REQUESTED, HoodieTimeline.COMPACTION_ACTION, "100"), dfs.getConf()); // Inflight Compaction HoodieTestDataGenerator.createCompactionAuxiliaryMetadata(basePath, - new HoodieInstant(State.INFLIGHT, COMPACTION_ACTION, "100"), dfs.getConf()); + new HoodieInstant(State.INFLIGHT, HoodieTimeline.COMPACTION_ACTION, "100"), dfs.getConf()); HoodieTestDataGenerator.createCommitFile(basePath, "100", dfs.getConf()); // Requested Compaction HoodieTestDataGenerator.createCompactionAuxiliaryMetadata(basePath, - new HoodieInstant(State.REQUESTED, COMPACTION_ACTION, "101"), dfs.getConf()); + new HoodieInstant(State.REQUESTED, HoodieTimeline.COMPACTION_ACTION, "101"), dfs.getConf()); // Inflight Compaction HoodieTestDataGenerator.createCompactionAuxiliaryMetadata(basePath, - new HoodieInstant(State.INFLIGHT, COMPACTION_ACTION, "101"), dfs.getConf()); + new HoodieInstant(State.INFLIGHT, HoodieTimeline.COMPACTION_ACTION, "101"), dfs.getConf()); HoodieTestDataGenerator.createCommitFile(basePath, "101", dfs.getConf()); // Requested Compaction HoodieTestDataGenerator.createCompactionAuxiliaryMetadata(basePath, - new HoodieInstant(State.REQUESTED, COMPACTION_ACTION, "102"), dfs.getConf()); + new HoodieInstant(State.REQUESTED, HoodieTimeline.COMPACTION_ACTION, "102"), dfs.getConf()); // Inflight Compaction HoodieTestDataGenerator.createCompactionAuxiliaryMetadata(basePath, - new HoodieInstant(State.INFLIGHT, COMPACTION_ACTION, "102"), dfs.getConf()); + new HoodieInstant(State.INFLIGHT, HoodieTimeline.COMPACTION_ACTION, "102"), dfs.getConf()); HoodieTestDataGenerator.createCommitFile(basePath, "102", dfs.getConf()); // Requested Compaction HoodieTestDataGenerator.createCompactionAuxiliaryMetadata(basePath, - new HoodieInstant(State.REQUESTED, COMPACTION_ACTION, "103"), dfs.getConf()); + new HoodieInstant(State.REQUESTED, HoodieTimeline.COMPACTION_ACTION, "103"), dfs.getConf()); // Inflight Compaction HoodieTestDataGenerator.createCompactionAuxiliaryMetadata(basePath, - new HoodieInstant(State.INFLIGHT, COMPACTION_ACTION, "103"), dfs.getConf()); + new HoodieInstant(State.INFLIGHT, HoodieTimeline.COMPACTION_ACTION, "103"), dfs.getConf()); HoodieTestDataGenerator.createCommitFile(basePath, "103", dfs.getConf()); HoodieTimeline timeline = metaClient.getActiveTimeline().getCommitsTimeline().filterCompletedInstants(); @@ -327,21 +327,21 @@ public class TestHoodieCommitArchiveLog { HoodieActiveTimeline.VALID_EXTENSIONS_IN_ACTIVE_TIMELINE); assertEquals("Should not delete any aux compaction files when maxCommitsToKeep is 5", 8, instants.size()); assertTrue("Requested Compaction must be present for 100", instants.contains( - new HoodieInstant(State.REQUESTED, COMPACTION_ACTION, "100"))); + new HoodieInstant(State.REQUESTED, HoodieTimeline.COMPACTION_ACTION, "100"))); assertTrue("Inflight Compaction must be present for 100", instants.contains( - new HoodieInstant(State.INFLIGHT, COMPACTION_ACTION, "100"))); + new HoodieInstant(State.INFLIGHT, HoodieTimeline.COMPACTION_ACTION, "100"))); assertTrue("Requested Compaction must be present for 101", instants.contains( - new HoodieInstant(State.REQUESTED, COMPACTION_ACTION, "101"))); + new HoodieInstant(State.REQUESTED, HoodieTimeline.COMPACTION_ACTION, "101"))); assertTrue("Inflight Compaction must be present for 101", instants.contains( - new HoodieInstant(State.INFLIGHT, COMPACTION_ACTION, "101"))); + new HoodieInstant(State.INFLIGHT, HoodieTimeline.COMPACTION_ACTION, "101"))); assertTrue("Requested Compaction must be present for 102", instants.contains( - new HoodieInstant(State.REQUESTED, COMPACTION_ACTION, "102"))); + new HoodieInstant(State.REQUESTED, HoodieTimeline.COMPACTION_ACTION, "102"))); assertTrue("Inflight Compaction must be present for 102", instants.contains( - new HoodieInstant(State.INFLIGHT, COMPACTION_ACTION, "102"))); + new HoodieInstant(State.INFLIGHT, HoodieTimeline.COMPACTION_ACTION, "102"))); assertTrue("Requested Compaction must be present for 103", instants.contains( - new HoodieInstant(State.REQUESTED, COMPACTION_ACTION, "103"))); + new HoodieInstant(State.REQUESTED, HoodieTimeline.COMPACTION_ACTION, "103"))); assertTrue("Inflight Compaction must be present for 103", instants.contains( - new HoodieInstant(State.INFLIGHT, COMPACTION_ACTION, "103"))); + new HoodieInstant(State.INFLIGHT, HoodieTimeline.COMPACTION_ACTION, "103"))); } @Test @@ -413,12 +413,12 @@ public class TestHoodieCommitArchiveLog { HoodieTestDataGenerator.createCommitFile(basePath, "100", dfs.getConf()); HoodieTestDataGenerator.createCompactionRequestedFile(basePath, "101", dfs.getConf()); HoodieTestDataGenerator.createCompactionAuxiliaryMetadata(basePath, - new HoodieInstant(State.REQUESTED, COMPACTION_ACTION, "101"), dfs.getConf()); + new HoodieInstant(State.REQUESTED, HoodieTimeline.COMPACTION_ACTION, "101"), dfs.getConf()); HoodieTestDataGenerator.createCommitFile(basePath, "102", dfs.getConf()); HoodieTestDataGenerator.createCommitFile(basePath, "103", dfs.getConf()); HoodieTestDataGenerator.createCompactionRequestedFile(basePath, "104", dfs.getConf()); HoodieTestDataGenerator.createCompactionAuxiliaryMetadata(basePath, - new HoodieInstant(State.REQUESTED, COMPACTION_ACTION, "104"), dfs.getConf()); + new HoodieInstant(State.REQUESTED, HoodieTimeline.COMPACTION_ACTION, "104"), dfs.getConf()); HoodieTestDataGenerator.createCommitFile(basePath, "105", dfs.getConf()); HoodieTestDataGenerator.createCommitFile(basePath, "106", dfs.getConf()); HoodieTestDataGenerator.createCommitFile(basePath, "107", dfs.getConf()); diff --git a/hoodie-client/src/test/java/com/uber/hoodie/io/TestHoodieCompactor.java b/hudi-client/src/test/java/org/apache/hudi/io/TestHoodieCompactor.java similarity index 87% rename from hoodie-client/src/test/java/com/uber/hoodie/io/TestHoodieCompactor.java rename to hudi-client/src/test/java/org/apache/hudi/io/TestHoodieCompactor.java index fade5d854..03b8d2195 100644 --- a/hoodie-client/src/test/java/com/uber/hoodie/io/TestHoodieCompactor.java +++ b/hudi-client/src/test/java/org/apache/hudi/io/TestHoodieCompactor.java @@ -16,39 +16,39 @@ * limitations under the License. */ -package com.uber.hoodie.io; +package org.apache.hudi.io; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; -import com.uber.hoodie.HoodieWriteClient; -import com.uber.hoodie.WriteStatus; -import com.uber.hoodie.common.HoodieClientTestUtils; -import com.uber.hoodie.common.HoodieTestDataGenerator; -import com.uber.hoodie.common.model.FileSlice; -import com.uber.hoodie.common.model.HoodieRecord; -import com.uber.hoodie.common.model.HoodieTableType; -import com.uber.hoodie.common.model.HoodieTestUtils; -import com.uber.hoodie.common.table.HoodieTableMetaClient; -import com.uber.hoodie.common.table.timeline.HoodieActiveTimeline; -import com.uber.hoodie.common.util.FSUtils; -import com.uber.hoodie.config.HoodieCompactionConfig; -import com.uber.hoodie.config.HoodieIndexConfig; -import com.uber.hoodie.config.HoodieMemoryConfig; -import com.uber.hoodie.config.HoodieStorageConfig; -import com.uber.hoodie.config.HoodieWriteConfig; -import com.uber.hoodie.exception.HoodieNotSupportedException; -import com.uber.hoodie.index.HoodieIndex; -import com.uber.hoodie.index.bloom.HoodieBloomIndex; -import com.uber.hoodie.io.compact.HoodieCompactor; -import com.uber.hoodie.io.compact.HoodieRealtimeTableCompactor; -import com.uber.hoodie.table.HoodieTable; import java.io.File; import java.io.IOException; import java.util.List; import java.util.stream.Collectors; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; +import org.apache.hudi.HoodieWriteClient; +import org.apache.hudi.WriteStatus; +import org.apache.hudi.common.HoodieClientTestUtils; +import org.apache.hudi.common.HoodieTestDataGenerator; +import org.apache.hudi.common.model.FileSlice; +import org.apache.hudi.common.model.HoodieRecord; +import org.apache.hudi.common.model.HoodieTableType; +import org.apache.hudi.common.model.HoodieTestUtils; +import org.apache.hudi.common.table.HoodieTableMetaClient; +import org.apache.hudi.common.table.timeline.HoodieActiveTimeline; +import org.apache.hudi.common.util.FSUtils; +import org.apache.hudi.config.HoodieCompactionConfig; +import org.apache.hudi.config.HoodieIndexConfig; +import org.apache.hudi.config.HoodieMemoryConfig; +import org.apache.hudi.config.HoodieStorageConfig; +import org.apache.hudi.config.HoodieWriteConfig; +import org.apache.hudi.exception.HoodieNotSupportedException; +import org.apache.hudi.index.HoodieIndex; +import org.apache.hudi.index.bloom.HoodieBloomIndex; +import org.apache.hudi.io.compact.HoodieCompactor; +import org.apache.hudi.io.compact.HoodieRealtimeTableCompactor; +import org.apache.hudi.table.HoodieTable; import org.apache.spark.api.java.JavaRDD; import org.apache.spark.api.java.JavaSparkContext; import org.junit.After; diff --git a/hoodie-client/src/test/java/com/uber/hoodie/io/TestHoodieMergeHandle.java b/hudi-client/src/test/java/org/apache/hudi/io/TestHoodieMergeHandle.java similarity index 92% rename from hoodie-client/src/test/java/com/uber/hoodie/io/TestHoodieMergeHandle.java rename to hudi-client/src/test/java/org/apache/hudi/io/TestHoodieMergeHandle.java index 44fa0e21a..1c2179ed5 100644 --- a/hoodie-client/src/test/java/com/uber/hoodie/io/TestHoodieMergeHandle.java +++ b/hudi-client/src/test/java/org/apache/hudi/io/TestHoodieMergeHandle.java @@ -16,34 +16,34 @@ * limitations under the License. */ -package com.uber.hoodie.io; +package org.apache.hudi.io; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotEquals; -import com.uber.hoodie.HoodieWriteClient; -import com.uber.hoodie.WriteStatus; -import com.uber.hoodie.common.HoodieClientTestUtils; -import com.uber.hoodie.common.HoodieTestDataGenerator; -import com.uber.hoodie.common.model.HoodieRecord; -import com.uber.hoodie.common.model.HoodieTestUtils; -import com.uber.hoodie.common.model.HoodieWriteStat; -import com.uber.hoodie.common.table.HoodieTableMetaClient; -import com.uber.hoodie.common.table.HoodieTimeline; -import com.uber.hoodie.common.table.timeline.HoodieActiveTimeline; -import com.uber.hoodie.common.util.FSUtils; -import com.uber.hoodie.config.HoodieCompactionConfig; -import com.uber.hoodie.config.HoodieIndexConfig; -import com.uber.hoodie.config.HoodieStorageConfig; -import com.uber.hoodie.config.HoodieWriteConfig; -import com.uber.hoodie.index.HoodieIndex; -import com.uber.hoodie.table.HoodieTable; import java.io.File; import java.io.IOException; import java.util.ArrayList; import java.util.List; import org.apache.hadoop.fs.FileSystem; +import org.apache.hudi.HoodieWriteClient; +import org.apache.hudi.WriteStatus; +import org.apache.hudi.common.HoodieClientTestUtils; +import org.apache.hudi.common.HoodieTestDataGenerator; +import org.apache.hudi.common.model.HoodieRecord; +import org.apache.hudi.common.model.HoodieTestUtils; +import org.apache.hudi.common.model.HoodieWriteStat; +import org.apache.hudi.common.table.HoodieTableMetaClient; +import org.apache.hudi.common.table.HoodieTimeline; +import org.apache.hudi.common.table.timeline.HoodieActiveTimeline; +import org.apache.hudi.common.util.FSUtils; +import org.apache.hudi.config.HoodieCompactionConfig; +import org.apache.hudi.config.HoodieIndexConfig; +import org.apache.hudi.config.HoodieStorageConfig; +import org.apache.hudi.config.HoodieWriteConfig; +import org.apache.hudi.index.HoodieIndex; +import org.apache.hudi.table.HoodieTable; import org.apache.spark.api.java.JavaRDD; import org.apache.spark.api.java.JavaSparkContext; import org.apache.spark.sql.Dataset; @@ -142,7 +142,7 @@ public class TestHoodieMergeHandle { HoodieTableMetaClient metaClient = new HoodieTableMetaClient(jsc.hadoopConfiguration(), basePath); HoodieTimeline timeline = new HoodieActiveTimeline(metaClient).getCommitTimeline(); assertEquals("Expecting a single commit.", 1, timeline.findInstantsAfter("000", Integer.MAX_VALUE).countInstants()); - assertEquals("Latest commit should be 001", newCommitTime, timeline.lastInstant().get().getTimestamp()); + Assert.assertEquals("Latest commit should be 001", newCommitTime, timeline.lastInstant().get().getTimestamp()); assertEquals("Must contain 44 records", records.size(), HoodieClientTestUtils.readCommit(basePath, sqlContext, timeline, newCommitTime).count()); @@ -170,7 +170,7 @@ public class TestHoodieMergeHandle { timeline = new HoodieActiveTimeline(metaClient).getCommitTimeline(); assertEquals("Expecting two commits.", 2, timeline.findInstantsAfter("000", Integer.MAX_VALUE) .countInstants()); - assertEquals("Latest commit should be 002", newCommitTime, timeline.lastInstant().get().getTimestamp()); + Assert.assertEquals("Latest commit should be 002", newCommitTime, timeline.lastInstant().get().getTimestamp()); Dataset dataSet = getRecords(); assertEquals("Must contain 45 records", 45, dataSet.count()); @@ -194,7 +194,7 @@ public class TestHoodieMergeHandle { timeline = new HoodieActiveTimeline(metaClient).getCommitTimeline(); assertEquals("Expecting three commits.", 3, timeline.findInstantsAfter("000", Integer.MAX_VALUE) .countInstants()); - assertEquals("Latest commit should be 003", newCommitTime, timeline.lastInstant().get().getTimestamp()); + Assert.assertEquals("Latest commit should be 003", newCommitTime, timeline.lastInstant().get().getTimestamp()); dataSet = getRecords(); assertEquals("Must contain 47 records", 47, dataSet.count()); @@ -227,7 +227,7 @@ public class TestHoodieMergeHandle { timeline = new HoodieActiveTimeline(metaClient).getCommitTimeline(); assertEquals("Expecting four commits.", 4, timeline.findInstantsAfter("000", Integer.MAX_VALUE) .countInstants()); - assertEquals("Latest commit should be 004", timeline.lastInstant().get().getTimestamp(), newCommitTime); + Assert.assertEquals("Latest commit should be 004", timeline.lastInstant().get().getTimestamp(), newCommitTime); // Check the entire dataset has 47 records still dataSet = getRecords(); diff --git a/hoodie-client/src/test/java/com/uber/hoodie/io/strategy/TestHoodieCompactionStrategy.java b/hudi-client/src/test/java/org/apache/hudi/io/strategy/TestHoodieCompactionStrategy.java similarity index 94% rename from hoodie-client/src/test/java/com/uber/hoodie/io/strategy/TestHoodieCompactionStrategy.java rename to hudi-client/src/test/java/org/apache/hudi/io/strategy/TestHoodieCompactionStrategy.java index 064eabe24..c3e123c94 100644 --- a/hoodie-client/src/test/java/com/uber/hoodie/io/strategy/TestHoodieCompactionStrategy.java +++ b/hudi-client/src/test/java/org/apache/hudi/io/strategy/TestHoodieCompactionStrategy.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package com.uber.hoodie.io.strategy; +package org.apache.hudi.io.strategy; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -24,19 +24,6 @@ import static org.junit.Assert.assertTrue; import com.beust.jcommander.internal.Lists; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Maps; -import com.uber.hoodie.avro.model.HoodieCompactionOperation; -import com.uber.hoodie.common.model.HoodieDataFile; -import com.uber.hoodie.common.model.HoodieLogFile; -import com.uber.hoodie.common.util.Option; -import com.uber.hoodie.common.util.collection.Pair; -import com.uber.hoodie.config.HoodieCompactionConfig; -import com.uber.hoodie.config.HoodieWriteConfig; -import com.uber.hoodie.io.compact.strategy.BoundedIOCompactionStrategy; -import com.uber.hoodie.io.compact.strategy.BoundedPartitionAwareCompactionStrategy; -import com.uber.hoodie.io.compact.strategy.DayBasedCompactionStrategy; -import com.uber.hoodie.io.compact.strategy.LogFileSizeBasedCompactionStrategy; -import com.uber.hoodie.io.compact.strategy.UnBoundedCompactionStrategy; -import com.uber.hoodie.io.compact.strategy.UnBoundedPartitionAwareCompactionStrategy; import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Date; @@ -45,6 +32,19 @@ import java.util.Map; import java.util.Random; import java.util.stream.Collectors; import org.apache.commons.lang3.time.DateUtils; +import org.apache.hudi.avro.model.HoodieCompactionOperation; +import org.apache.hudi.common.model.HoodieDataFile; +import org.apache.hudi.common.model.HoodieLogFile; +import org.apache.hudi.common.util.Option; +import org.apache.hudi.common.util.collection.Pair; +import org.apache.hudi.config.HoodieCompactionConfig; +import org.apache.hudi.config.HoodieWriteConfig; +import org.apache.hudi.io.compact.strategy.BoundedIOCompactionStrategy; +import org.apache.hudi.io.compact.strategy.BoundedPartitionAwareCompactionStrategy; +import org.apache.hudi.io.compact.strategy.DayBasedCompactionStrategy; +import org.apache.hudi.io.compact.strategy.LogFileSizeBasedCompactionStrategy; +import org.apache.hudi.io.compact.strategy.UnBoundedCompactionStrategy; +import org.apache.hudi.io.compact.strategy.UnBoundedPartitionAwareCompactionStrategy; import org.junit.Assert; import org.junit.Test; diff --git a/hoodie-client/src/test/java/com/uber/hoodie/io/strategy/TestHoodieDataFile.java b/hudi-client/src/test/java/org/apache/hudi/io/strategy/TestHoodieDataFile.java similarity index 94% rename from hoodie-client/src/test/java/com/uber/hoodie/io/strategy/TestHoodieDataFile.java rename to hudi-client/src/test/java/org/apache/hudi/io/strategy/TestHoodieDataFile.java index 92006586b..62a9fa791 100644 --- a/hoodie-client/src/test/java/com/uber/hoodie/io/strategy/TestHoodieDataFile.java +++ b/hudi-client/src/test/java/org/apache/hudi/io/strategy/TestHoodieDataFile.java @@ -16,10 +16,10 @@ * limitations under the License. */ -package com.uber.hoodie.io.strategy; +package org.apache.hudi.io.strategy; -import com.uber.hoodie.common.model.HoodieDataFile; import java.util.UUID; +import org.apache.hudi.common.model.HoodieDataFile; public class TestHoodieDataFile extends HoodieDataFile { diff --git a/hoodie-client/src/test/java/com/uber/hoodie/io/strategy/TestHoodieLogFile.java b/hudi-client/src/test/java/org/apache/hudi/io/strategy/TestHoodieLogFile.java similarity index 93% rename from hoodie-client/src/test/java/com/uber/hoodie/io/strategy/TestHoodieLogFile.java rename to hudi-client/src/test/java/org/apache/hudi/io/strategy/TestHoodieLogFile.java index c7cc77576..c8b5df93d 100644 --- a/hoodie-client/src/test/java/com/uber/hoodie/io/strategy/TestHoodieLogFile.java +++ b/hudi-client/src/test/java/org/apache/hudi/io/strategy/TestHoodieLogFile.java @@ -16,10 +16,10 @@ * limitations under the License. */ -package com.uber.hoodie.io.strategy; +package org.apache.hudi.io.strategy; -import com.uber.hoodie.common.model.HoodieLogFile; import org.apache.hadoop.fs.Path; +import org.apache.hudi.common.model.HoodieLogFile; public class TestHoodieLogFile extends HoodieLogFile { diff --git a/hoodie-client/src/test/java/com/uber/hoodie/metrics/TestHoodieMetrics.java b/hudi-client/src/test/java/org/apache/hudi/metrics/TestHoodieMetrics.java similarity index 91% rename from hoodie-client/src/test/java/com/uber/hoodie/metrics/TestHoodieMetrics.java rename to hudi-client/src/test/java/org/apache/hudi/metrics/TestHoodieMetrics.java index 9adaa982b..1a1b40758 100644 --- a/hoodie-client/src/test/java/com/uber/hoodie/metrics/TestHoodieMetrics.java +++ b/hudi-client/src/test/java/org/apache/hudi/metrics/TestHoodieMetrics.java @@ -16,15 +16,15 @@ * limitations under the License. */ -package com.uber.hoodie.metrics; +package org.apache.hudi.metrics; -import static com.uber.hoodie.metrics.Metrics.registerGauge; +import static org.apache.hudi.metrics.Metrics.registerGauge; import static org.junit.Assert.assertTrue; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; -import com.uber.hoodie.config.HoodieWriteConfig; import org.apache.commons.configuration.ConfigurationException; +import org.apache.hudi.config.HoodieWriteConfig; import org.junit.Before; import org.junit.Test; diff --git a/hoodie-client/src/test/java/com/uber/hoodie/table/TestCopyOnWriteTable.java b/hudi-client/src/test/java/org/apache/hudi/table/TestCopyOnWriteTable.java similarity index 95% rename from hoodie-client/src/test/java/com/uber/hoodie/table/TestCopyOnWriteTable.java rename to hudi-client/src/test/java/org/apache/hudi/table/TestCopyOnWriteTable.java index 306f9683f..980c15848 100644 --- a/hoodie-client/src/test/java/com/uber/hoodie/table/TestCopyOnWriteTable.java +++ b/hudi-client/src/test/java/org/apache/hudi/table/TestCopyOnWriteTable.java @@ -16,34 +16,13 @@ * limitations under the License. */ -package com.uber.hoodie.table; +package org.apache.hudi.table; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; -import com.uber.hoodie.WriteStatus; -import com.uber.hoodie.common.BloomFilter; -import com.uber.hoodie.common.HoodieClientTestUtils; -import com.uber.hoodie.common.HoodieTestDataGenerator; -import com.uber.hoodie.common.TestRawTripPayload; -import com.uber.hoodie.common.TestRawTripPayload.MetadataMergeWriteStatus; -import com.uber.hoodie.common.model.HoodieKey; -import com.uber.hoodie.common.model.HoodieRecord; -import com.uber.hoodie.common.model.HoodieRecordLocation; -import com.uber.hoodie.common.model.HoodieTestUtils; -import com.uber.hoodie.common.table.HoodieTableMetaClient; -import com.uber.hoodie.common.table.HoodieTimeline; -import com.uber.hoodie.common.util.FSUtils; -import com.uber.hoodie.common.util.Option; -import com.uber.hoodie.common.util.ParquetUtils; -import com.uber.hoodie.common.util.collection.Pair; -import com.uber.hoodie.config.HoodieCompactionConfig; -import com.uber.hoodie.config.HoodieStorageConfig; -import com.uber.hoodie.config.HoodieWriteConfig; -import com.uber.hoodie.io.HoodieCreateHandle; -import com.uber.hoodie.table.HoodieCopyOnWriteTable.UpsertPartitioner; import java.io.File; import java.util.ArrayList; import java.util.Arrays; @@ -53,6 +32,27 @@ import java.util.UUID; import org.apache.avro.generic.GenericRecord; import org.apache.commons.io.IOUtils; import org.apache.hadoop.fs.Path; +import org.apache.hudi.WriteStatus; +import org.apache.hudi.common.BloomFilter; +import org.apache.hudi.common.HoodieClientTestUtils; +import org.apache.hudi.common.HoodieTestDataGenerator; +import org.apache.hudi.common.TestRawTripPayload; +import org.apache.hudi.common.TestRawTripPayload.MetadataMergeWriteStatus; +import org.apache.hudi.common.model.HoodieKey; +import org.apache.hudi.common.model.HoodieRecord; +import org.apache.hudi.common.model.HoodieRecordLocation; +import org.apache.hudi.common.model.HoodieTestUtils; +import org.apache.hudi.common.table.HoodieTableMetaClient; +import org.apache.hudi.common.table.HoodieTimeline; +import org.apache.hudi.common.util.FSUtils; +import org.apache.hudi.common.util.Option; +import org.apache.hudi.common.util.ParquetUtils; +import org.apache.hudi.common.util.collection.Pair; +import org.apache.hudi.config.HoodieCompactionConfig; +import org.apache.hudi.config.HoodieStorageConfig; +import org.apache.hudi.config.HoodieWriteConfig; +import org.apache.hudi.io.HoodieCreateHandle; +import org.apache.hudi.table.HoodieCopyOnWriteTable.UpsertPartitioner; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; import org.apache.parquet.avro.AvroReadSupport; diff --git a/hoodie-client/src/test/java/com/uber/hoodie/table/TestMergeOnReadTable.java b/hudi-client/src/test/java/org/apache/hudi/table/TestMergeOnReadTable.java similarity index 95% rename from hoodie-client/src/test/java/com/uber/hoodie/table/TestMergeOnReadTable.java rename to hudi-client/src/test/java/org/apache/hudi/table/TestMergeOnReadTable.java index 2857f12e6..98a5d621a 100644 --- a/hoodie-client/src/test/java/com/uber/hoodie/table/TestMergeOnReadTable.java +++ b/hudi-client/src/test/java/org/apache/hudi/table/TestMergeOnReadTable.java @@ -16,46 +16,13 @@ * limitations under the License. */ -package com.uber.hoodie.table; +package org.apache.hudi.table; -import static com.uber.hoodie.common.HoodieTestDataGenerator.TRIP_EXAMPLE_SCHEMA; +import static org.apache.hudi.common.HoodieTestDataGenerator.TRIP_EXAMPLE_SCHEMA; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; -import com.uber.hoodie.HoodieReadClient; -import com.uber.hoodie.HoodieWriteClient; -import com.uber.hoodie.WriteStatus; -import com.uber.hoodie.common.HoodieClientTestUtils; -import com.uber.hoodie.common.HoodieMergeOnReadTestUtils; -import com.uber.hoodie.common.HoodieTestDataGenerator; -import com.uber.hoodie.common.TestRawTripPayload.MetadataMergeWriteStatus; -import com.uber.hoodie.common.minicluster.HdfsTestService; -import com.uber.hoodie.common.model.FileSlice; -import com.uber.hoodie.common.model.HoodieCommitMetadata; -import com.uber.hoodie.common.model.HoodieDataFile; -import com.uber.hoodie.common.model.HoodieFileGroup; -import com.uber.hoodie.common.model.HoodieKey; -import com.uber.hoodie.common.model.HoodieRecord; -import com.uber.hoodie.common.model.HoodieRollingStat; -import com.uber.hoodie.common.model.HoodieRollingStatMetadata; -import com.uber.hoodie.common.model.HoodieTableType; -import com.uber.hoodie.common.model.HoodieTestUtils; -import com.uber.hoodie.common.table.HoodieTableMetaClient; -import com.uber.hoodie.common.table.HoodieTimeline; -import com.uber.hoodie.common.table.SyncableFileSystemView; -import com.uber.hoodie.common.table.TableFileSystemView; -import com.uber.hoodie.common.table.timeline.HoodieActiveTimeline; -import com.uber.hoodie.common.table.timeline.HoodieInstant; -import com.uber.hoodie.common.table.timeline.HoodieInstant.State; -import com.uber.hoodie.common.table.view.HoodieTableFileSystemView; -import com.uber.hoodie.common.util.Option; -import com.uber.hoodie.config.HoodieCompactionConfig; -import com.uber.hoodie.config.HoodieIndexConfig; -import com.uber.hoodie.config.HoodieStorageConfig; -import com.uber.hoodie.config.HoodieWriteConfig; -import com.uber.hoodie.index.HoodieIndex; -import com.uber.hoodie.index.HoodieIndex.IndexType; import java.io.File; import java.io.IOException; import java.util.ArrayList; @@ -71,6 +38,40 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.hudi.HoodieReadClient; +import org.apache.hudi.HoodieWriteClient; +import org.apache.hudi.WriteStatus; +import org.apache.hudi.common.HoodieClientTestUtils; +import org.apache.hudi.common.HoodieMergeOnReadTestUtils; +import org.apache.hudi.common.HoodieTestDataGenerator; +import org.apache.hudi.common.TestRawTripPayload.MetadataMergeWriteStatus; +import org.apache.hudi.common.minicluster.HdfsTestService; +import org.apache.hudi.common.model.FileSlice; +import org.apache.hudi.common.model.HoodieCommitMetadata; +import org.apache.hudi.common.model.HoodieDataFile; +import org.apache.hudi.common.model.HoodieFileGroup; +import org.apache.hudi.common.model.HoodieKey; +import org.apache.hudi.common.model.HoodieRecord; +import org.apache.hudi.common.model.HoodieRollingStat; +import org.apache.hudi.common.model.HoodieRollingStatMetadata; +import org.apache.hudi.common.model.HoodieTableType; +import org.apache.hudi.common.model.HoodieTestUtils; +import org.apache.hudi.common.table.HoodieTableMetaClient; +import org.apache.hudi.common.table.HoodieTimeline; +import org.apache.hudi.common.table.SyncableFileSystemView; +import org.apache.hudi.common.table.TableFileSystemView.ReadOptimizedView; +import org.apache.hudi.common.table.TableFileSystemView.RealtimeView; +import org.apache.hudi.common.table.timeline.HoodieActiveTimeline; +import org.apache.hudi.common.table.timeline.HoodieInstant; +import org.apache.hudi.common.table.timeline.HoodieInstant.State; +import org.apache.hudi.common.table.view.HoodieTableFileSystemView; +import org.apache.hudi.common.util.Option; +import org.apache.hudi.config.HoodieCompactionConfig; +import org.apache.hudi.config.HoodieIndexConfig; +import org.apache.hudi.config.HoodieStorageConfig; +import org.apache.hudi.config.HoodieWriteConfig; +import org.apache.hudi.index.HoodieIndex; +import org.apache.hudi.index.HoodieIndex.IndexType; import org.apache.spark.api.java.JavaRDD; import org.apache.spark.api.java.JavaSparkContext; import org.apache.spark.sql.SQLContext; @@ -184,13 +185,13 @@ public class TestMergeOnReadTable { Option deltaCommit = metaClient.getActiveTimeline().getDeltaCommitTimeline().firstInstant(); assertTrue(deltaCommit.isPresent()); - assertEquals("Delta commit should be 001", "001", deltaCommit.get().getTimestamp()); + Assert.assertEquals("Delta commit should be 001", "001", deltaCommit.get().getTimestamp()); Option commit = metaClient.getActiveTimeline().getCommitTimeline().firstInstant(); assertFalse(commit.isPresent()); FileStatus[] allFiles = HoodieTestUtils.listAllDataFilesInPath(metaClient.getFs(), cfg.getBasePath()); - TableFileSystemView.ReadOptimizedView roView = new HoodieTableFileSystemView(metaClient, + ReadOptimizedView roView = new HoodieTableFileSystemView(metaClient, metaClient.getCommitTimeline().filterCompletedInstants(), allFiles); Stream dataFilesToRead = roView.getLatestDataFiles(); assertTrue(!dataFilesToRead.findAny().isPresent()); @@ -297,7 +298,7 @@ public class TestMergeOnReadTable { assertFalse(commit.isPresent()); FileStatus[] allFiles = HoodieTestUtils.listAllDataFilesInPath(metaClient.getFs(), cfg.getBasePath()); - TableFileSystemView.ReadOptimizedView roView = new HoodieTableFileSystemView(metaClient, + ReadOptimizedView roView = new HoodieTableFileSystemView(metaClient, metaClient.getCommitTimeline().filterCompletedInstants(), allFiles); Stream dataFilesToRead = roView.getLatestDataFiles(); assertTrue(!dataFilesToRead.findAny().isPresent()); @@ -444,7 +445,7 @@ public class TestMergeOnReadTable { assertFalse(commit.isPresent()); FileStatus[] allFiles = HoodieTestUtils.listAllDataFilesInPath(metaClient.getFs(), cfg.getBasePath()); - TableFileSystemView.ReadOptimizedView roView = new HoodieTableFileSystemView(metaClient, + ReadOptimizedView roView = new HoodieTableFileSystemView(metaClient, metaClient.getCommitTimeline().filterCompletedInstants(), allFiles); Stream dataFilesToRead = roView.getLatestDataFiles(); assertTrue(!dataFilesToRead.findAny().isPresent()); @@ -609,7 +610,7 @@ public class TestMergeOnReadTable { assertFalse(commit.isPresent()); FileStatus[] allFiles = HoodieTestUtils.listAllDataFilesInPath(metaClient.getFs(), cfg.getBasePath()); - TableFileSystemView.ReadOptimizedView roView = new HoodieTableFileSystemView(metaClient, + ReadOptimizedView roView = new HoodieTableFileSystemView(metaClient, metaClient.getCommitTimeline().filterCompletedInstants(), allFiles); Stream dataFilesToRead = roView.getLatestDataFiles(); assertTrue(!dataFilesToRead.findAny().isPresent()); @@ -733,7 +734,7 @@ public class TestMergeOnReadTable { metaClient.getCommitTimeline().filterCompletedInstants(), allFiles); dataFilesToRead = roView.getLatestDataFiles(); assertTrue(!dataFilesToRead.findAny().isPresent()); - HoodieTableFileSystemView.RealtimeView rtView = new HoodieTableFileSystemView(metaClient, + RealtimeView rtView = new HoodieTableFileSystemView(metaClient, metaClient.getCommitTimeline().filterCompletedInstants(), allFiles); List fileGroups = ((HoodieTableFileSystemView) rtView).getAllFileGroups().collect(Collectors .toList()); @@ -781,7 +782,7 @@ public class TestMergeOnReadTable { assertFalse(commit.isPresent()); FileStatus[] allFiles = HoodieTestUtils.listAllDataFilesInPath(metaClient.getFs(), cfg.getBasePath()); - TableFileSystemView.ReadOptimizedView roView = new HoodieTableFileSystemView(metaClient, + ReadOptimizedView roView = new HoodieTableFileSystemView(metaClient, metaClient.getCommitsTimeline().filterCompletedInstants(), allFiles); Stream dataFilesToRead = roView.getLatestDataFiles(); Map parquetFileIdToSize = dataFilesToRead.collect( @@ -968,7 +969,7 @@ public class TestMergeOnReadTable { HoodieTable table = HoodieTable .getHoodieTable(new HoodieTableMetaClient(jsc.hadoopConfiguration(), basePath), config, jsc); - TableFileSystemView.RealtimeView tableRTFileSystemView = table.getRTFileSystemView(); + RealtimeView tableRTFileSystemView = table.getRTFileSystemView(); long numLogFiles = 0; for (String partitionPath : dataGen.getPartitionPaths()) { @@ -1029,7 +1030,7 @@ public class TestMergeOnReadTable { writeClient.rollback(newCommitTime); final HoodieTableMetaClient metaClient = new HoodieTableMetaClient(jsc.hadoopConfiguration(), basePath); HoodieTable table = HoodieTable.getHoodieTable(metaClient, config, jsc); - TableFileSystemView.RealtimeView tableRTFileSystemView = table.getRTFileSystemView(); + RealtimeView tableRTFileSystemView = table.getRTFileSystemView(); long numLogFiles = 0; for (String partitionPath : dataGen.getPartitionPaths()) { @@ -1063,7 +1064,7 @@ public class TestMergeOnReadTable { HoodieTable table = HoodieTable .getHoodieTable(new HoodieTableMetaClient(jsc.hadoopConfiguration(), basePath), config, jsc); - TableFileSystemView.RealtimeView tableRTFileSystemView = table.getRTFileSystemView(); + RealtimeView tableRTFileSystemView = table.getRTFileSystemView(); long numLogFiles = 0; for (String partitionPath : dataGen.getPartitionPaths()) { diff --git a/hoodie-client/src/test/resources/exampleEvolvedSchema.txt b/hudi-client/src/test/resources/exampleEvolvedSchema.txt similarity index 100% rename from hoodie-client/src/test/resources/exampleEvolvedSchema.txt rename to hudi-client/src/test/resources/exampleEvolvedSchema.txt diff --git a/hoodie-client/src/test/resources/exampleSchema.txt b/hudi-client/src/test/resources/exampleSchema.txt similarity index 100% rename from hoodie-client/src/test/resources/exampleSchema.txt rename to hudi-client/src/test/resources/exampleSchema.txt diff --git a/hoodie-client/src/test/resources/log4j-surefire-quiet.properties b/hudi-client/src/test/resources/log4j-surefire-quiet.properties similarity index 100% rename from hoodie-client/src/test/resources/log4j-surefire-quiet.properties rename to hudi-client/src/test/resources/log4j-surefire-quiet.properties diff --git a/hoodie-client/src/test/resources/log4j-surefire.properties b/hudi-client/src/test/resources/log4j-surefire.properties similarity index 100% rename from hoodie-client/src/test/resources/log4j-surefire.properties rename to hudi-client/src/test/resources/log4j-surefire.properties diff --git a/hoodie-common/pom.xml b/hudi-common/pom.xml similarity index 97% rename from hoodie-common/pom.xml rename to hudi-common/pom.xml index e9ece6b46..fdde6022f 100644 --- a/hoodie-common/pom.xml +++ b/hudi-common/pom.xml @@ -17,13 +17,13 @@ --> - hoodie - com.uber.hoodie - 0.4.8-SNAPSHOT + hudi + org.apache.hudi + 0.5.0-SNAPSHOT 4.0.0 - hoodie-common + hudi-common ${project.basedir}/src/main/resources/META-INF diff --git a/hoodie-common/src/main/avro/HoodieArchivedMetaEntry.avsc b/hudi-common/src/main/avro/HoodieArchivedMetaEntry.avsc similarity index 95% rename from hoodie-common/src/main/avro/HoodieArchivedMetaEntry.avsc rename to hudi-common/src/main/avro/HoodieArchivedMetaEntry.avsc index 86376518d..a504e7c94 100644 --- a/hoodie-common/src/main/avro/HoodieArchivedMetaEntry.avsc +++ b/hudi-common/src/main/avro/HoodieArchivedMetaEntry.avsc @@ -1,7 +1,7 @@ { "type":"record", "name":"HoodieArchivedMetaEntry", - "namespace":"com.uber.hoodie.avro.model", + "namespace":"org.apache.hudi.avro.model", "fields":[ { "name":"hoodieCommitMetadata", @@ -52,4 +52,4 @@ "type":["null","string"] } ] -} \ No newline at end of file +} diff --git a/hoodie-common/src/main/avro/HoodieCleanMetadata.avsc b/hudi-common/src/main/avro/HoodieCleanMetadata.avsc similarity index 95% rename from hoodie-common/src/main/avro/HoodieCleanMetadata.avsc rename to hudi-common/src/main/avro/HoodieCleanMetadata.avsc index 3e55a97e9..594b6c727 100644 --- a/hoodie-common/src/main/avro/HoodieCleanMetadata.avsc +++ b/hudi-common/src/main/avro/HoodieCleanMetadata.avsc @@ -1,4 +1,4 @@ -{"namespace": "com.uber.hoodie.avro.model", +{"namespace": "org.apache.hudi.avro.model", "type": "record", "name": "HoodieCleanMetadata", "fields": [ diff --git a/hoodie-common/src/main/avro/HoodieCommitMetadata.avsc b/hudi-common/src/main/avro/HoodieCommitMetadata.avsc similarity index 98% rename from hoodie-common/src/main/avro/HoodieCommitMetadata.avsc rename to hudi-common/src/main/avro/HoodieCommitMetadata.avsc index c5e8ce31b..db4cacc55 100644 --- a/hoodie-common/src/main/avro/HoodieCommitMetadata.avsc +++ b/hudi-common/src/main/avro/HoodieCommitMetadata.avsc @@ -1,5 +1,5 @@ { - "namespace":"com.uber.hoodie.avro.model", + "namespace":"org.apache.hudi.avro.model", "type":"record", "name":"HoodieCommitMetadata", "fields":[ diff --git a/hoodie-common/src/main/avro/HoodieCompactionMetadata.avsc b/hudi-common/src/main/avro/HoodieCompactionMetadata.avsc similarity index 96% rename from hoodie-common/src/main/avro/HoodieCompactionMetadata.avsc rename to hudi-common/src/main/avro/HoodieCompactionMetadata.avsc index 871b1b513..cff87264d 100644 --- a/hoodie-common/src/main/avro/HoodieCompactionMetadata.avsc +++ b/hudi-common/src/main/avro/HoodieCompactionMetadata.avsc @@ -1,5 +1,5 @@ { - "namespace":"com.uber.hoodie.avro.model", + "namespace":"org.apache.hudi.avro.model", "type":"record", "name":"HoodieCompactionMetadata", "fields":[ @@ -39,4 +39,4 @@ }] } ] -} \ No newline at end of file +} diff --git a/hoodie-common/src/main/avro/HoodieCompactionOperation.avsc b/hudi-common/src/main/avro/HoodieCompactionOperation.avsc similarity index 97% rename from hoodie-common/src/main/avro/HoodieCompactionOperation.avsc rename to hudi-common/src/main/avro/HoodieCompactionOperation.avsc index 450b89898..37f9af417 100644 --- a/hoodie-common/src/main/avro/HoodieCompactionOperation.avsc +++ b/hudi-common/src/main/avro/HoodieCompactionOperation.avsc @@ -1,5 +1,5 @@ { - "namespace":"com.uber.hoodie.avro.model", + "namespace":"org.apache.hudi.avro.model", "type":"record", "name":"HoodieCompactionPlan", "fields":[ diff --git a/hoodie-common/src/main/avro/HoodieRestoreMetadata.avsc b/hudi-common/src/main/avro/HoodieRestoreMetadata.avsc similarity index 91% rename from hoodie-common/src/main/avro/HoodieRestoreMetadata.avsc rename to hudi-common/src/main/avro/HoodieRestoreMetadata.avsc index a96c12d92..dfd8c547f 100644 --- a/hoodie-common/src/main/avro/HoodieRestoreMetadata.avsc +++ b/hudi-common/src/main/avro/HoodieRestoreMetadata.avsc @@ -1,4 +1,4 @@ -{"namespace": "com.uber.hoodie.avro.model", +{"namespace": "org.apache.hudi.avro.model", "type": "record", "name": "HoodieRestoreMetadata", "fields": [ diff --git a/hoodie-common/src/main/avro/HoodieRollbackMetadata.avsc b/hudi-common/src/main/avro/HoodieRollbackMetadata.avsc similarity index 94% rename from hoodie-common/src/main/avro/HoodieRollbackMetadata.avsc rename to hudi-common/src/main/avro/HoodieRollbackMetadata.avsc index 60d2d63a3..a9f91ac3e 100644 --- a/hoodie-common/src/main/avro/HoodieRollbackMetadata.avsc +++ b/hudi-common/src/main/avro/HoodieRollbackMetadata.avsc @@ -1,4 +1,4 @@ -{"namespace": "com.uber.hoodie.avro.model", +{"namespace": "org.apache.hudi.avro.model", "type": "record", "name": "HoodieRollbackMetadata", "fields": [ diff --git a/hoodie-common/src/main/avro/HoodieSavePointMetadata.avsc b/hudi-common/src/main/avro/HoodieSavePointMetadata.avsc similarity index 93% rename from hoodie-common/src/main/avro/HoodieSavePointMetadata.avsc rename to hudi-common/src/main/avro/HoodieSavePointMetadata.avsc index 6fc92246a..be377f2dd 100644 --- a/hoodie-common/src/main/avro/HoodieSavePointMetadata.avsc +++ b/hudi-common/src/main/avro/HoodieSavePointMetadata.avsc @@ -1,4 +1,4 @@ -{"namespace": "com.uber.hoodie.avro.model", +{"namespace": "org.apache.hudi.avro.model", "type": "record", "name": "HoodieSavepointMetadata", "fields": [ diff --git a/hoodie-common/src/main/java/com/uber/hoodie/avro/HoodieAvroWriteSupport.java b/hudi-common/src/main/java/org/apache/hudi/avro/HoodieAvroWriteSupport.java similarity index 95% rename from hoodie-common/src/main/java/com/uber/hoodie/avro/HoodieAvroWriteSupport.java rename to hudi-common/src/main/java/org/apache/hudi/avro/HoodieAvroWriteSupport.java index aa0eabdaf..8a0164781 100644 --- a/hoodie-common/src/main/java/com/uber/hoodie/avro/HoodieAvroWriteSupport.java +++ b/hudi-common/src/main/java/org/apache/hudi/avro/HoodieAvroWriteSupport.java @@ -16,11 +16,11 @@ * limitations under the License. */ -package com.uber.hoodie.avro; +package org.apache.hudi.avro; -import com.uber.hoodie.common.BloomFilter; import java.util.HashMap; import org.apache.avro.Schema; +import org.apache.hudi.common.BloomFilter; import org.apache.parquet.avro.AvroWriteSupport; import org.apache.parquet.hadoop.api.WriteSupport; import org.apache.parquet.schema.MessageType; @@ -36,7 +36,7 @@ public class HoodieAvroWriteSupport extends AvroWriteSupport { public static final String HOODIE_AVRO_BLOOM_FILTER_METADATA_KEY = - "com.uber.hoodie.bloomfilter"; + "org.apache.hudi.bloomfilter"; public static final String HOODIE_MIN_RECORD_KEY_FOOTER = "hoodie_min_record_key"; public static final String HOODIE_MAX_RECORD_KEY_FOOTER = "hoodie_max_record_key"; diff --git a/hoodie-common/src/main/java/com/uber/hoodie/avro/MercifulJsonConverter.java b/hudi-common/src/main/java/org/apache/hudi/avro/MercifulJsonConverter.java similarity index 99% rename from hoodie-common/src/main/java/com/uber/hoodie/avro/MercifulJsonConverter.java rename to hudi-common/src/main/java/org/apache/hudi/avro/MercifulJsonConverter.java index 6a62a5ead..e2a8cc72d 100644 --- a/hoodie-common/src/main/java/com/uber/hoodie/avro/MercifulJsonConverter.java +++ b/hudi-common/src/main/java/org/apache/hudi/avro/MercifulJsonConverter.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package com.uber.hoodie.avro; +package org.apache.hudi.avro; import com.fasterxml.jackson.databind.ObjectMapper; import java.io.IOException; diff --git a/hoodie-common/src/main/java/com/uber/hoodie/common/BloomFilter.java b/hudi-common/src/main/java/org/apache/hudi/common/BloomFilter.java similarity index 97% rename from hoodie-common/src/main/java/com/uber/hoodie/common/BloomFilter.java rename to hudi-common/src/main/java/org/apache/hudi/common/BloomFilter.java index b768834e7..5b18ef7c7 100644 --- a/hoodie-common/src/main/java/com/uber/hoodie/common/BloomFilter.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/BloomFilter.java @@ -16,9 +16,8 @@ * limitations under the License. */ -package com.uber.hoodie.common; +package org.apache.hudi.common; -import com.uber.hoodie.exception.HoodieIndexException; import java.io.ByteArrayInputStream; import java.io.DataInputStream; import java.io.DataOutputStream; @@ -28,6 +27,7 @@ import javax.xml.bind.DatatypeConverter; import org.apache.commons.io.output.ByteArrayOutputStream; import org.apache.hadoop.util.bloom.Key; import org.apache.hadoop.util.hash.Hash; +import org.apache.hudi.exception.HoodieIndexException; /** * A Bloom filter implementation built on top of {@link org.apache.hadoop.util.bloom.BloomFilter}. diff --git a/hoodie-common/src/main/java/com/uber/hoodie/common/HoodieCleanStat.java b/hudi-common/src/main/java/org/apache/hudi/common/HoodieCleanStat.java similarity index 95% rename from hoodie-common/src/main/java/com/uber/hoodie/common/HoodieCleanStat.java rename to hudi-common/src/main/java/org/apache/hudi/common/HoodieCleanStat.java index 3e3aebb53..00252635e 100644 --- a/hoodie-common/src/main/java/com/uber/hoodie/common/HoodieCleanStat.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/HoodieCleanStat.java @@ -16,13 +16,13 @@ * limitations under the License. */ -package com.uber.hoodie.common; +package org.apache.hudi.common; -import com.uber.hoodie.common.model.HoodieCleaningPolicy; -import com.uber.hoodie.common.table.timeline.HoodieInstant; -import com.uber.hoodie.common.util.Option; import java.io.Serializable; import java.util.List; +import org.apache.hudi.common.model.HoodieCleaningPolicy; +import org.apache.hudi.common.table.timeline.HoodieInstant; +import org.apache.hudi.common.util.Option; /** * Collects stats about a single partition clean operation diff --git a/hoodie-common/src/main/java/com/uber/hoodie/common/HoodieJsonPayload.java b/hudi-common/src/main/java/org/apache/hudi/common/HoodieJsonPayload.java similarity index 94% rename from hoodie-common/src/main/java/com/uber/hoodie/common/HoodieJsonPayload.java rename to hudi-common/src/main/java/org/apache/hudi/common/HoodieJsonPayload.java index ddbf5d6fe..9f3eb93f8 100644 --- a/hoodie-common/src/main/java/com/uber/hoodie/common/HoodieJsonPayload.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/HoodieJsonPayload.java @@ -16,14 +16,10 @@ * limitations under the License. */ -package com.uber.hoodie.common; +package org.apache.hudi.common; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; -import com.uber.hoodie.avro.MercifulJsonConverter; -import com.uber.hoodie.common.model.HoodieRecordPayload; -import com.uber.hoodie.common.util.Option; -import com.uber.hoodie.exception.HoodieException; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; @@ -34,6 +30,10 @@ import java.util.zip.InflaterInputStream; import org.apache.avro.Schema; import org.apache.avro.generic.IndexedRecord; import org.apache.commons.io.IOUtils; +import org.apache.hudi.avro.MercifulJsonConverter; +import org.apache.hudi.common.model.HoodieRecordPayload; +import org.apache.hudi.common.util.Option; +import org.apache.hudi.exception.HoodieException; public class HoodieJsonPayload implements HoodieRecordPayload { diff --git a/hoodie-common/src/main/java/com/uber/hoodie/common/HoodieRollbackStat.java b/hudi-common/src/main/java/org/apache/hudi/common/HoodieRollbackStat.java similarity index 99% rename from hoodie-common/src/main/java/com/uber/hoodie/common/HoodieRollbackStat.java rename to hudi-common/src/main/java/org/apache/hudi/common/HoodieRollbackStat.java index b43bad8e1..ae6e1cc22 100644 --- a/hoodie-common/src/main/java/com/uber/hoodie/common/HoodieRollbackStat.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/HoodieRollbackStat.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package com.uber.hoodie.common; +package org.apache.hudi.common; import java.io.Serializable; import java.util.List; diff --git a/hoodie-common/src/main/java/com/uber/hoodie/common/SerializableConfiguration.java b/hudi-common/src/main/java/org/apache/hudi/common/SerializableConfiguration.java similarity index 98% rename from hoodie-common/src/main/java/com/uber/hoodie/common/SerializableConfiguration.java rename to hudi-common/src/main/java/org/apache/hudi/common/SerializableConfiguration.java index 04732e28c..0d5dc6cfe 100644 --- a/hoodie-common/src/main/java/com/uber/hoodie/common/SerializableConfiguration.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/SerializableConfiguration.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package com.uber.hoodie.common; +package org.apache.hudi.common; import java.io.IOException; import java.io.ObjectInputStream; diff --git a/hoodie-common/src/main/java/com/uber/hoodie/common/io/storage/HoodieWrapperFileSystem.java b/hudi-common/src/main/java/org/apache/hudi/common/io/storage/HoodieWrapperFileSystem.java similarity index 98% rename from hoodie-common/src/main/java/com/uber/hoodie/common/io/storage/HoodieWrapperFileSystem.java rename to hudi-common/src/main/java/org/apache/hudi/common/io/storage/HoodieWrapperFileSystem.java index a234a3b74..d9ff91485 100644 --- a/hoodie-common/src/main/java/com/uber/hoodie/common/io/storage/HoodieWrapperFileSystem.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/io/storage/HoodieWrapperFileSystem.java @@ -16,14 +16,8 @@ * limitations under the License. */ -package com.uber.hoodie.common.io.storage; +package org.apache.hudi.common.io.storage; -import com.uber.hoodie.common.storage.StorageSchemes; -import com.uber.hoodie.common.util.ConsistencyGuard; -import com.uber.hoodie.common.util.FSUtils; -import com.uber.hoodie.common.util.NoOpConsistencyGuard; -import com.uber.hoodie.exception.HoodieException; -import com.uber.hoodie.exception.HoodieIOException; import java.io.IOException; import java.net.URI; import java.net.URISyntaxException; @@ -57,6 +51,12 @@ import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.security.Credentials; import org.apache.hadoop.security.token.Token; import org.apache.hadoop.util.Progressable; +import org.apache.hudi.common.storage.StorageSchemes; +import org.apache.hudi.common.util.ConsistencyGuard; +import org.apache.hudi.common.util.FSUtils; +import org.apache.hudi.common.util.NoOpConsistencyGuard; +import org.apache.hudi.exception.HoodieException; +import org.apache.hudi.exception.HoodieIOException; /** * HoodieWrapperFileSystem wraps the default file system. It holds state about the open streams in diff --git a/hoodie-common/src/main/java/com/uber/hoodie/common/io/storage/SizeAwareFSDataOutputStream.java b/hudi-common/src/main/java/org/apache/hudi/common/io/storage/SizeAwareFSDataOutputStream.java similarity index 94% rename from hoodie-common/src/main/java/com/uber/hoodie/common/io/storage/SizeAwareFSDataOutputStream.java rename to hudi-common/src/main/java/org/apache/hudi/common/io/storage/SizeAwareFSDataOutputStream.java index 9ea2db061..9a8f3edcc 100644 --- a/hoodie-common/src/main/java/com/uber/hoodie/common/io/storage/SizeAwareFSDataOutputStream.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/io/storage/SizeAwareFSDataOutputStream.java @@ -16,15 +16,15 @@ * limitations under the License. */ -package com.uber.hoodie.common.io.storage; +package org.apache.hudi.common.io.storage; -import com.uber.hoodie.common.util.ConsistencyGuard; -import com.uber.hoodie.exception.HoodieException; import java.io.IOException; import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicLong; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.Path; +import org.apache.hudi.common.util.ConsistencyGuard; +import org.apache.hudi.exception.HoodieException; /** * Wrapper over FSDataOutputStream to keep track of the size of the written bytes. This diff --git a/hoodie-common/src/main/java/com/uber/hoodie/common/model/ActionType.java b/hudi-common/src/main/java/org/apache/hudi/common/model/ActionType.java similarity index 95% rename from hoodie-common/src/main/java/com/uber/hoodie/common/model/ActionType.java rename to hudi-common/src/main/java/org/apache/hudi/common/model/ActionType.java index 1850b838d..9f69dfd2d 100644 --- a/hoodie-common/src/main/java/com/uber/hoodie/common/model/ActionType.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/model/ActionType.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package com.uber.hoodie.common.model; +package org.apache.hudi.common.model; public enum ActionType { commit, savepoint, compaction, clean, rollback diff --git a/hoodie-common/src/main/java/com/uber/hoodie/common/model/CompactionOperation.java b/hudi-common/src/main/java/org/apache/hudi/common/model/CompactionOperation.java similarity index 96% rename from hoodie-common/src/main/java/com/uber/hoodie/common/model/CompactionOperation.java rename to hudi-common/src/main/java/org/apache/hudi/common/model/CompactionOperation.java index b69159896..ff6c210ad 100644 --- a/hoodie-common/src/main/java/com/uber/hoodie/common/model/CompactionOperation.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/model/CompactionOperation.java @@ -16,11 +16,8 @@ * limitations under the License. */ -package com.uber.hoodie.common.model; +package org.apache.hudi.common.model; -import com.uber.hoodie.avro.model.HoodieCompactionOperation; -import com.uber.hoodie.common.util.FSUtils; -import com.uber.hoodie.common.util.Option; import java.io.Serializable; import java.util.ArrayList; import java.util.HashMap; @@ -29,6 +26,9 @@ import java.util.Map; import java.util.Objects; import java.util.stream.Collectors; import org.apache.hadoop.fs.Path; +import org.apache.hudi.avro.model.HoodieCompactionOperation; +import org.apache.hudi.common.util.FSUtils; +import org.apache.hudi.common.util.Option; /** * Encapsulates all the needed information about a compaction and make a decision whether this diff --git a/hoodie-common/src/main/java/com/uber/hoodie/common/model/FileSlice.java b/hudi-common/src/main/java/org/apache/hudi/common/model/FileSlice.java similarity index 98% rename from hoodie-common/src/main/java/com/uber/hoodie/common/model/FileSlice.java rename to hudi-common/src/main/java/org/apache/hudi/common/model/FileSlice.java index 305f33704..567f9b673 100644 --- a/hoodie-common/src/main/java/com/uber/hoodie/common/model/FileSlice.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/model/FileSlice.java @@ -16,13 +16,13 @@ * limitations under the License. */ -package com.uber.hoodie.common.model; +package org.apache.hudi.common.model; -import com.uber.hoodie.common.util.Option; import java.io.Serializable; import java.util.Objects; import java.util.TreeSet; import java.util.stream.Stream; +import org.apache.hudi.common.util.Option; /** * Within a file group, a slice is a combination of data file written at a commit time and list of diff --git a/hoodie-common/src/main/java/com/uber/hoodie/common/model/HoodieArchivedLogFile.java b/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieArchivedLogFile.java similarity index 97% rename from hoodie-common/src/main/java/com/uber/hoodie/common/model/HoodieArchivedLogFile.java rename to hudi-common/src/main/java/org/apache/hudi/common/model/HoodieArchivedLogFile.java index 46ba7e05a..895959d7d 100644 --- a/hoodie-common/src/main/java/com/uber/hoodie/common/model/HoodieArchivedLogFile.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieArchivedLogFile.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package com.uber.hoodie.common.model; +package org.apache.hudi.common.model; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.Path; diff --git a/hoodie-common/src/main/java/com/uber/hoodie/common/model/HoodieAvroPayload.java b/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieAvroPayload.java similarity index 92% rename from hoodie-common/src/main/java/com/uber/hoodie/common/model/HoodieAvroPayload.java rename to hudi-common/src/main/java/org/apache/hudi/common/model/HoodieAvroPayload.java index a606912dd..7aefc51c6 100644 --- a/hoodie-common/src/main/java/com/uber/hoodie/common/model/HoodieAvroPayload.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieAvroPayload.java @@ -16,15 +16,15 @@ * limitations under the License. */ -package com.uber.hoodie.common.model; +package org.apache.hudi.common.model; -import com.uber.hoodie.common.util.HoodieAvroUtils; -import com.uber.hoodie.common.util.Option; -import com.uber.hoodie.exception.HoodieIOException; import java.io.IOException; import org.apache.avro.Schema; import org.apache.avro.generic.GenericRecord; import org.apache.avro.generic.IndexedRecord; +import org.apache.hudi.common.util.HoodieAvroUtils; +import org.apache.hudi.common.util.Option; +import org.apache.hudi.exception.HoodieIOException; /** * This is a payload to wrap a existing Hoodie Avro Record. Useful to create a HoodieRecord over diff --git a/hoodie-common/src/main/java/com/uber/hoodie/common/model/HoodieCleaningPolicy.java b/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieCleaningPolicy.java similarity index 95% rename from hoodie-common/src/main/java/com/uber/hoodie/common/model/HoodieCleaningPolicy.java rename to hudi-common/src/main/java/org/apache/hudi/common/model/HoodieCleaningPolicy.java index c49617d62..36d9c3b2a 100644 --- a/hoodie-common/src/main/java/com/uber/hoodie/common/model/HoodieCleaningPolicy.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieCleaningPolicy.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package com.uber.hoodie.common.model; +package org.apache.hudi.common.model; public enum HoodieCleaningPolicy { KEEP_LATEST_FILE_VERSIONS, diff --git a/hoodie-common/src/main/java/com/uber/hoodie/common/model/HoodieCommitMetadata.java b/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieCommitMetadata.java similarity index 99% rename from hoodie-common/src/main/java/com/uber/hoodie/common/model/HoodieCommitMetadata.java rename to hudi-common/src/main/java/org/apache/hudi/common/model/HoodieCommitMetadata.java index b76ba46ce..b48d39527 100644 --- a/hoodie-common/src/main/java/com/uber/hoodie/common/model/HoodieCommitMetadata.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieCommitMetadata.java @@ -16,14 +16,13 @@ * limitations under the License. */ -package com.uber.hoodie.common.model; +package org.apache.hudi.common.model; import com.fasterxml.jackson.annotation.JsonAutoDetect; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.PropertyAccessor; import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.ObjectMapper; -import com.uber.hoodie.common.util.FSUtils; import java.io.IOException; import java.io.Serializable; import java.nio.charset.Charset; @@ -31,6 +30,7 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; +import org.apache.hudi.common.util.FSUtils; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; diff --git a/hoodie-common/src/main/java/com/uber/hoodie/common/model/HoodieDataFile.java b/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieDataFile.java similarity index 96% rename from hoodie-common/src/main/java/com/uber/hoodie/common/model/HoodieDataFile.java rename to hudi-common/src/main/java/org/apache/hudi/common/model/HoodieDataFile.java index c554f6a02..8a34abbb0 100644 --- a/hoodie-common/src/main/java/com/uber/hoodie/common/model/HoodieDataFile.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieDataFile.java @@ -16,13 +16,13 @@ * limitations under the License. */ -package com.uber.hoodie.common.model; +package org.apache.hudi.common.model; -import com.uber.hoodie.common.util.FSUtils; import java.io.Serializable; import java.util.Objects; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.Path; +import org.apache.hudi.common.util.FSUtils; public class HoodieDataFile implements Serializable { diff --git a/hoodie-common/src/main/java/com/uber/hoodie/common/model/HoodieDeltaWriteStat.java b/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieDeltaWriteStat.java similarity index 97% rename from hoodie-common/src/main/java/com/uber/hoodie/common/model/HoodieDeltaWriteStat.java rename to hudi-common/src/main/java/org/apache/hudi/common/model/HoodieDeltaWriteStat.java index bece38c81..1b7dcb78d 100644 --- a/hoodie-common/src/main/java/com/uber/hoodie/common/model/HoodieDeltaWriteStat.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieDeltaWriteStat.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package com.uber.hoodie.common.model; +package org.apache.hudi.common.model; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; diff --git a/hoodie-common/src/main/java/com/uber/hoodie/common/model/HoodieFileFormat.java b/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieFileFormat.java similarity index 96% rename from hoodie-common/src/main/java/com/uber/hoodie/common/model/HoodieFileFormat.java rename to hudi-common/src/main/java/org/apache/hudi/common/model/HoodieFileFormat.java index f3ce5e393..fc1247a31 100644 --- a/hoodie-common/src/main/java/com/uber/hoodie/common/model/HoodieFileFormat.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieFileFormat.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package com.uber.hoodie.common.model; +package org.apache.hudi.common.model; public enum HoodieFileFormat { PARQUET(".parquet"), HOODIE_LOG(".log"); diff --git a/hoodie-common/src/main/java/com/uber/hoodie/common/model/HoodieFileGroup.java b/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieFileGroup.java similarity index 96% rename from hoodie-common/src/main/java/com/uber/hoodie/common/model/HoodieFileGroup.java rename to hudi-common/src/main/java/org/apache/hudi/common/model/HoodieFileGroup.java index d06e112dc..9763b9e4a 100644 --- a/hoodie-common/src/main/java/com/uber/hoodie/common/model/HoodieFileGroup.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieFileGroup.java @@ -16,14 +16,17 @@ * limitations under the License. */ -package com.uber.hoodie.common.model; +package org.apache.hudi.common.model; -import com.uber.hoodie.common.table.HoodieTimeline; -import com.uber.hoodie.common.table.timeline.HoodieInstant; -import com.uber.hoodie.common.util.Option; import java.io.Serializable; -import java.util.*; +import java.util.Comparator; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; import java.util.stream.Stream; +import org.apache.hudi.common.table.HoodieTimeline; +import org.apache.hudi.common.table.timeline.HoodieInstant; +import org.apache.hudi.common.util.Option; /** * A set of data/base files + set of log files, that make up an unit for all operations diff --git a/hoodie-common/src/main/java/com/uber/hoodie/common/model/HoodieFileGroupId.java b/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieFileGroupId.java similarity index 98% rename from hoodie-common/src/main/java/com/uber/hoodie/common/model/HoodieFileGroupId.java rename to hudi-common/src/main/java/org/apache/hudi/common/model/HoodieFileGroupId.java index e4af28b9a..b396de0eb 100644 --- a/hoodie-common/src/main/java/com/uber/hoodie/common/model/HoodieFileGroupId.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieFileGroupId.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package com.uber.hoodie.common.model; +package org.apache.hudi.common.model; import java.io.Serializable; import java.util.Objects; diff --git a/hoodie-common/src/main/java/com/uber/hoodie/common/model/HoodieKey.java b/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieKey.java similarity index 98% rename from hoodie-common/src/main/java/com/uber/hoodie/common/model/HoodieKey.java rename to hudi-common/src/main/java/org/apache/hudi/common/model/HoodieKey.java index 17efa2a05..0b124a557 100644 --- a/hoodie-common/src/main/java/com/uber/hoodie/common/model/HoodieKey.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieKey.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package com.uber.hoodie.common.model; +package org.apache.hudi.common.model; import com.google.common.base.Objects; import java.io.Serializable; diff --git a/hoodie-common/src/main/java/com/uber/hoodie/common/model/HoodieLogFile.java b/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieLogFile.java similarity index 98% rename from hoodie-common/src/main/java/com/uber/hoodie/common/model/HoodieLogFile.java rename to hudi-common/src/main/java/org/apache/hudi/common/model/HoodieLogFile.java index 46ebe2720..a5445932f 100644 --- a/hoodie-common/src/main/java/com/uber/hoodie/common/model/HoodieLogFile.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieLogFile.java @@ -16,9 +16,8 @@ * limitations under the License. */ -package com.uber.hoodie.common.model; +package org.apache.hudi.common.model; -import com.uber.hoodie.common.util.FSUtils; import java.io.IOException; import java.io.Serializable; import java.util.Comparator; @@ -26,6 +25,7 @@ import java.util.Objects; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hudi.common.util.FSUtils; /** * Abstracts a single log file. Contains methods to extract metadata like the fileId, version and extension from the log diff --git a/hoodie-common/src/main/java/com/uber/hoodie/common/model/HoodiePartitionMetadata.java b/hudi-common/src/main/java/org/apache/hudi/common/model/HoodiePartitionMetadata.java similarity index 98% rename from hoodie-common/src/main/java/com/uber/hoodie/common/model/HoodiePartitionMetadata.java rename to hudi-common/src/main/java/org/apache/hudi/common/model/HoodiePartitionMetadata.java index 6aa08b8cf..d3a45099f 100644 --- a/hoodie-common/src/main/java/com/uber/hoodie/common/model/HoodiePartitionMetadata.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/model/HoodiePartitionMetadata.java @@ -16,15 +16,15 @@ * limitations under the License. */ -package com.uber.hoodie.common.model; +package org.apache.hudi.common.model; -import com.uber.hoodie.exception.HoodieException; import java.io.IOException; import java.util.Properties; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hudi.exception.HoodieException; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; diff --git a/hoodie-common/src/main/java/com/uber/hoodie/common/model/HoodieRecord.java b/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieRecord.java similarity index 98% rename from hoodie-common/src/main/java/com/uber/hoodie/common/model/HoodieRecord.java rename to hudi-common/src/main/java/org/apache/hudi/common/model/HoodieRecord.java index 0deadb300..c03f175d6 100644 --- a/hoodie-common/src/main/java/com/uber/hoodie/common/model/HoodieRecord.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieRecord.java @@ -16,13 +16,13 @@ * limitations under the License. */ -package com.uber.hoodie.common.model; +package org.apache.hudi.common.model; import com.google.common.base.Objects; import com.google.common.collect.ImmutableList; -import com.uber.hoodie.common.util.Option; import java.io.Serializable; import java.util.List; +import org.apache.hudi.common.util.Option; /** * A Single Record managed by Hoodie TODO - Make this generic diff --git a/hoodie-common/src/main/java/com/uber/hoodie/common/model/HoodieRecordLocation.java b/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieRecordLocation.java similarity index 98% rename from hoodie-common/src/main/java/com/uber/hoodie/common/model/HoodieRecordLocation.java rename to hudi-common/src/main/java/org/apache/hudi/common/model/HoodieRecordLocation.java index 75f41ef1f..ef1e630a8 100644 --- a/hoodie-common/src/main/java/com/uber/hoodie/common/model/HoodieRecordLocation.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieRecordLocation.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package com.uber.hoodie.common.model; +package org.apache.hudi.common.model; import com.google.common.base.Objects; import java.io.Serializable; diff --git a/hoodie-common/src/main/java/com/uber/hoodie/common/model/HoodieRecordPayload.java b/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieRecordPayload.java similarity index 97% rename from hoodie-common/src/main/java/com/uber/hoodie/common/model/HoodieRecordPayload.java rename to hudi-common/src/main/java/org/apache/hudi/common/model/HoodieRecordPayload.java index ca2eb0841..0fc7703ae 100644 --- a/hoodie-common/src/main/java/com/uber/hoodie/common/model/HoodieRecordPayload.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieRecordPayload.java @@ -16,14 +16,14 @@ * limitations under the License. */ -package com.uber.hoodie.common.model; +package org.apache.hudi.common.model; -import com.uber.hoodie.common.util.Option; import java.io.IOException; import java.io.Serializable; import java.util.Map; import org.apache.avro.Schema; import org.apache.avro.generic.IndexedRecord; +import org.apache.hudi.common.util.Option; /** * Every Hoodie dataset has an implementation of the HoodieRecordPayload This abstracts diff --git a/hoodie-common/src/main/java/com/uber/hoodie/common/model/HoodieRollingStat.java b/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieRollingStat.java similarity index 98% rename from hoodie-common/src/main/java/com/uber/hoodie/common/model/HoodieRollingStat.java rename to hudi-common/src/main/java/org/apache/hudi/common/model/HoodieRollingStat.java index e66dc1cc8..49c0db8a9 100644 --- a/hoodie-common/src/main/java/com/uber/hoodie/common/model/HoodieRollingStat.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieRollingStat.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package com.uber.hoodie.common.model; +package org.apache.hudi.common.model; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import java.io.Serializable; diff --git a/hoodie-common/src/main/java/com/uber/hoodie/common/model/HoodieRollingStatMetadata.java b/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieRollingStatMetadata.java similarity index 98% rename from hoodie-common/src/main/java/com/uber/hoodie/common/model/HoodieRollingStatMetadata.java rename to hudi-common/src/main/java/org/apache/hudi/common/model/HoodieRollingStatMetadata.java index 443445f69..f30a6157a 100644 --- a/hoodie-common/src/main/java/com/uber/hoodie/common/model/HoodieRollingStatMetadata.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieRollingStatMetadata.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package com.uber.hoodie.common.model; +package org.apache.hudi.common.model; import java.io.IOException; import java.io.Serializable; diff --git a/hoodie-common/src/main/java/com/uber/hoodie/common/model/HoodieTableType.java b/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieTableType.java similarity index 97% rename from hoodie-common/src/main/java/com/uber/hoodie/common/model/HoodieTableType.java rename to hudi-common/src/main/java/org/apache/hudi/common/model/HoodieTableType.java index 2b65c4826..9fbb01744 100644 --- a/hoodie-common/src/main/java/com/uber/hoodie/common/model/HoodieTableType.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieTableType.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package com.uber.hoodie.common.model; +package org.apache.hudi.common.model; /** * Type of the Hoodie Table. diff --git a/hoodie-common/src/main/java/com/uber/hoodie/common/model/HoodieWriteStat.java b/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieWriteStat.java similarity index 99% rename from hoodie-common/src/main/java/com/uber/hoodie/common/model/HoodieWriteStat.java rename to hudi-common/src/main/java/org/apache/hudi/common/model/HoodieWriteStat.java index a9a3217d5..ddc33ff0b 100644 --- a/hoodie-common/src/main/java/com/uber/hoodie/common/model/HoodieWriteStat.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieWriteStat.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package com.uber.hoodie.common.model; +package org.apache.hudi.common.model; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; diff --git a/hoodie-common/src/main/java/com/uber/hoodie/common/storage/SizeAwareDataInputStream.java b/hudi-common/src/main/java/org/apache/hudi/common/storage/SizeAwareDataInputStream.java similarity index 97% rename from hoodie-common/src/main/java/com/uber/hoodie/common/storage/SizeAwareDataInputStream.java rename to hudi-common/src/main/java/org/apache/hudi/common/storage/SizeAwareDataInputStream.java index fe2edc73f..da833b8a4 100644 --- a/hoodie-common/src/main/java/com/uber/hoodie/common/storage/SizeAwareDataInputStream.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/storage/SizeAwareDataInputStream.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package com.uber.hoodie.common.storage; +package org.apache.hudi.common.storage; import java.io.DataInputStream; import java.io.IOException; diff --git a/hoodie-common/src/main/java/com/uber/hoodie/common/storage/StorageSchemes.java b/hudi-common/src/main/java/org/apache/hudi/common/storage/StorageSchemes.java similarity index 98% rename from hoodie-common/src/main/java/com/uber/hoodie/common/storage/StorageSchemes.java rename to hudi-common/src/main/java/org/apache/hudi/common/storage/StorageSchemes.java index 4455790ab..5d6d762b5 100644 --- a/hoodie-common/src/main/java/com/uber/hoodie/common/storage/StorageSchemes.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/storage/StorageSchemes.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package com.uber.hoodie.common.storage; +package org.apache.hudi.common.storage; import java.util.Arrays; diff --git a/hoodie-common/src/main/java/com/uber/hoodie/common/table/HoodieTableConfig.java b/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableConfig.java similarity index 93% rename from hoodie-common/src/main/java/com/uber/hoodie/common/table/HoodieTableConfig.java rename to hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableConfig.java index e676832c6..6ddc1b348 100644 --- a/hoodie-common/src/main/java/com/uber/hoodie/common/table/HoodieTableConfig.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableConfig.java @@ -16,12 +16,8 @@ * limitations under the License. */ -package com.uber.hoodie.common.table; +package org.apache.hudi.common.table; -import com.uber.hoodie.common.model.HoodieAvroPayload; -import com.uber.hoodie.common.model.HoodieFileFormat; -import com.uber.hoodie.common.model.HoodieTableType; -import com.uber.hoodie.exception.HoodieIOException; import java.io.IOException; import java.io.Serializable; import java.util.Date; @@ -32,6 +28,10 @@ import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hudi.common.model.HoodieAvroPayload; +import org.apache.hudi.common.model.HoodieFileFormat; +import org.apache.hudi.common.model.HoodieTableType; +import org.apache.hudi.exception.HoodieIOException; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; @@ -137,7 +137,10 @@ public class HoodieTableConfig implements Serializable { * Read the payload class for HoodieRecords from the table properties */ public String getPayloadClass() { - return props.getProperty(HOODIE_PAYLOAD_CLASS_PROP_NAME, DEFAULT_PAYLOAD_CLASS); + // There could be datasets written with payload class from com.uber.hoodie. Need to transparently + // change to org.apache.hudi + return props.getProperty(HOODIE_PAYLOAD_CLASS_PROP_NAME, DEFAULT_PAYLOAD_CLASS) + .replace("com.uber.hoodie", "org.apache.hudi"); } /** diff --git a/hoodie-common/src/main/java/com/uber/hoodie/common/table/HoodieTableMetaClient.java b/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableMetaClient.java similarity index 95% rename from hoodie-common/src/main/java/com/uber/hoodie/common/table/HoodieTableMetaClient.java rename to hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableMetaClient.java index 7ed0214af..3d441a086 100644 --- a/hoodie-common/src/main/java/com/uber/hoodie/common/table/HoodieTableMetaClient.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableMetaClient.java @@ -16,23 +16,9 @@ * limitations under the License. */ -package com.uber.hoodie.common.table; - -import static com.uber.hoodie.common.model.HoodieTableType.MERGE_ON_READ; +package org.apache.hudi.common.table; import com.google.common.base.Preconditions; -import com.uber.hoodie.common.SerializableConfiguration; -import com.uber.hoodie.common.io.storage.HoodieWrapperFileSystem; -import com.uber.hoodie.common.model.HoodieTableType; -import com.uber.hoodie.common.table.timeline.HoodieActiveTimeline; -import com.uber.hoodie.common.table.timeline.HoodieArchivedTimeline; -import com.uber.hoodie.common.table.timeline.HoodieInstant; -import com.uber.hoodie.common.util.ConsistencyGuardConfig; -import com.uber.hoodie.common.util.FSUtils; -import com.uber.hoodie.common.util.FailSafeConsistencyGuard; -import com.uber.hoodie.common.util.NoOpConsistencyGuard; -import com.uber.hoodie.exception.DatasetNotFoundException; -import com.uber.hoodie.exception.HoodieException; import java.io.File; import java.io.IOException; import java.io.Serializable; @@ -48,6 +34,18 @@ import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.PathFilter; +import org.apache.hudi.common.SerializableConfiguration; +import org.apache.hudi.common.io.storage.HoodieWrapperFileSystem; +import org.apache.hudi.common.model.HoodieTableType; +import org.apache.hudi.common.table.timeline.HoodieActiveTimeline; +import org.apache.hudi.common.table.timeline.HoodieArchivedTimeline; +import org.apache.hudi.common.table.timeline.HoodieInstant; +import org.apache.hudi.common.util.ConsistencyGuardConfig; +import org.apache.hudi.common.util.FSUtils; +import org.apache.hudi.common.util.FailSafeConsistencyGuard; +import org.apache.hudi.common.util.NoOpConsistencyGuard; +import org.apache.hudi.exception.DatasetNotFoundException; +import org.apache.hudi.exception.HoodieException; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; @@ -286,7 +284,7 @@ public class HoodieTableMetaClient implements Serializable { Properties properties = new Properties(); properties.setProperty(HoodieTableConfig.HOODIE_TABLE_NAME_PROP_NAME, tableName); properties.setProperty(HoodieTableConfig.HOODIE_TABLE_TYPE_PROP_NAME, tableType.name()); - if (tableType == MERGE_ON_READ) { + if (tableType == HoodieTableType.MERGE_ON_READ) { properties.setProperty(HoodieTableConfig.HOODIE_PAYLOAD_CLASS_PROP_NAME, payloadClassName); } return HoodieTableMetaClient.initializePathAsHoodieDataset(hadoopConf, basePath, properties); diff --git a/hoodie-common/src/main/java/com/uber/hoodie/common/table/HoodieTimeline.java b/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTimeline.java similarity index 96% rename from hoodie-common/src/main/java/com/uber/hoodie/common/table/HoodieTimeline.java rename to hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTimeline.java index 759343352..2a024f29c 100644 --- a/hoodie-common/src/main/java/com/uber/hoodie/common/table/HoodieTimeline.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTimeline.java @@ -16,17 +16,17 @@ * limitations under the License. */ -package com.uber.hoodie.common.table; +package org.apache.hudi.common.table; -import com.uber.hoodie.common.table.timeline.HoodieDefaultTimeline; -import com.uber.hoodie.common.table.timeline.HoodieInstant; -import com.uber.hoodie.common.table.timeline.HoodieInstant.State; -import com.uber.hoodie.common.util.Option; -import com.uber.hoodie.common.util.StringUtils; import java.io.Serializable; import java.util.function.BiPredicate; import java.util.function.Predicate; import java.util.stream.Stream; +import org.apache.hudi.common.table.timeline.HoodieDefaultTimeline; +import org.apache.hudi.common.table.timeline.HoodieInstant; +import org.apache.hudi.common.table.timeline.HoodieInstant.State; +import org.apache.hudi.common.util.Option; +import org.apache.hudi.common.util.StringUtils; /** * HoodieTimeline is a view of meta-data instants in the hoodie dataset. Instants are specific @@ -34,7 +34,7 @@ import java.util.stream.Stream; * operations create new instance of timelines which filter on the instants and this can be * chained. * - * @see com.uber.hoodie.common.table.HoodieTableMetaClient + * @see HoodieTableMetaClient * @see HoodieDefaultTimeline * @see HoodieInstant * @since 0.3.0 diff --git a/hoodie-common/src/main/java/com/uber/hoodie/common/table/SyncableFileSystemView.java b/hudi-common/src/main/java/org/apache/hudi/common/table/SyncableFileSystemView.java similarity index 97% rename from hoodie-common/src/main/java/com/uber/hoodie/common/table/SyncableFileSystemView.java rename to hudi-common/src/main/java/org/apache/hudi/common/table/SyncableFileSystemView.java index 28df72ead..88f61e9c6 100644 --- a/hoodie-common/src/main/java/com/uber/hoodie/common/table/SyncableFileSystemView.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/SyncableFileSystemView.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package com.uber.hoodie.common.table; +package org.apache.hudi.common.table; /* * A consolidated file-system view interface exposing both realtime and read-optimized views along with diff --git a/hoodie-common/src/main/java/com/uber/hoodie/common/table/TableFileSystemView.java b/hudi-common/src/main/java/org/apache/hudi/common/table/TableFileSystemView.java similarity index 92% rename from hoodie-common/src/main/java/com/uber/hoodie/common/table/TableFileSystemView.java rename to hudi-common/src/main/java/org/apache/hudi/common/table/TableFileSystemView.java index 5605e3ae2..f5f04a706 100644 --- a/hoodie-common/src/main/java/com/uber/hoodie/common/table/TableFileSystemView.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/TableFileSystemView.java @@ -16,17 +16,17 @@ * limitations under the License. */ -package com.uber.hoodie.common.table; +package org.apache.hudi.common.table; -import com.uber.hoodie.common.model.CompactionOperation; -import com.uber.hoodie.common.model.FileSlice; -import com.uber.hoodie.common.model.HoodieDataFile; -import com.uber.hoodie.common.model.HoodieFileGroup; -import com.uber.hoodie.common.table.timeline.HoodieInstant; -import com.uber.hoodie.common.util.Option; -import com.uber.hoodie.common.util.collection.Pair; import java.util.List; import java.util.stream.Stream; +import org.apache.hudi.common.model.CompactionOperation; +import org.apache.hudi.common.model.FileSlice; +import org.apache.hudi.common.model.HoodieDataFile; +import org.apache.hudi.common.model.HoodieFileGroup; +import org.apache.hudi.common.table.timeline.HoodieInstant; +import org.apache.hudi.common.util.Option; +import org.apache.hudi.common.util.collection.Pair; /** * Interface for viewing the table file system. diff --git a/hoodie-common/src/main/java/com/uber/hoodie/common/table/log/AbstractHoodieLogRecordScanner.java b/hudi-common/src/main/java/org/apache/hudi/common/table/log/AbstractHoodieLogRecordScanner.java similarity index 94% rename from hoodie-common/src/main/java/com/uber/hoodie/common/table/log/AbstractHoodieLogRecordScanner.java rename to hudi-common/src/main/java/org/apache/hudi/common/table/log/AbstractHoodieLogRecordScanner.java index c2fe7301c..16d788f57 100644 --- a/hoodie-common/src/main/java/com/uber/hoodie/common/table/log/AbstractHoodieLogRecordScanner.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/log/AbstractHoodieLogRecordScanner.java @@ -16,23 +16,11 @@ * limitations under the License. */ -package com.uber.hoodie.common.table.log; +package org.apache.hudi.common.table.log; -import static com.uber.hoodie.common.table.log.block.HoodieLogBlock.HeaderMetadataType.INSTANT_TIME; -import static com.uber.hoodie.common.table.log.block.HoodieLogBlock.HoodieLogBlockType.CORRUPT_BLOCK; +import static org.apache.hudi.common.table.log.block.HoodieLogBlock.HeaderMetadataType.INSTANT_TIME; +import static org.apache.hudi.common.table.log.block.HoodieLogBlock.HoodieLogBlockType.CORRUPT_BLOCK; -import com.uber.hoodie.common.model.HoodieKey; -import com.uber.hoodie.common.model.HoodieLogFile; -import com.uber.hoodie.common.model.HoodieRecord; -import com.uber.hoodie.common.model.HoodieRecordPayload; -import com.uber.hoodie.common.table.HoodieTableMetaClient; -import com.uber.hoodie.common.table.HoodieTimeline; -import com.uber.hoodie.common.table.log.block.HoodieAvroDataBlock; -import com.uber.hoodie.common.table.log.block.HoodieCommandBlock; -import com.uber.hoodie.common.table.log.block.HoodieDeleteBlock; -import com.uber.hoodie.common.table.log.block.HoodieLogBlock; -import com.uber.hoodie.common.util.SpillableMapUtils; -import com.uber.hoodie.exception.HoodieIOException; import java.io.IOException; import java.util.ArrayDeque; import java.util.Arrays; @@ -47,6 +35,18 @@ import org.apache.avro.generic.GenericRecord; import org.apache.avro.generic.IndexedRecord; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hudi.common.model.HoodieKey; +import org.apache.hudi.common.model.HoodieLogFile; +import org.apache.hudi.common.model.HoodieRecord; +import org.apache.hudi.common.model.HoodieRecordPayload; +import org.apache.hudi.common.table.HoodieTableMetaClient; +import org.apache.hudi.common.table.HoodieTimeline; +import org.apache.hudi.common.table.log.block.HoodieAvroDataBlock; +import org.apache.hudi.common.table.log.block.HoodieCommandBlock; +import org.apache.hudi.common.table.log.block.HoodieDeleteBlock; +import org.apache.hudi.common.table.log.block.HoodieLogBlock; +import org.apache.hudi.common.util.SpillableMapUtils; +import org.apache.hudi.exception.HoodieIOException; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; diff --git a/hoodie-common/src/main/java/com/uber/hoodie/common/table/log/HoodieLogFileReader.java b/hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieLogFileReader.java similarity index 94% rename from hoodie-common/src/main/java/com/uber/hoodie/common/table/log/HoodieLogFileReader.java rename to hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieLogFileReader.java index 13b53e83d..b479bfd79 100644 --- a/hoodie-common/src/main/java/com/uber/hoodie/common/table/log/HoodieLogFileReader.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieLogFileReader.java @@ -16,22 +16,9 @@ * limitations under the License. */ -package com.uber.hoodie.common.table.log; +package org.apache.hudi.common.table.log; import com.google.common.base.Preconditions; -import com.uber.hoodie.common.model.HoodieLogFile; -import com.uber.hoodie.common.table.log.block.HoodieAvroDataBlock; -import com.uber.hoodie.common.table.log.block.HoodieCommandBlock; -import com.uber.hoodie.common.table.log.block.HoodieCorruptBlock; -import com.uber.hoodie.common.table.log.block.HoodieDeleteBlock; -import com.uber.hoodie.common.table.log.block.HoodieLogBlock; -import com.uber.hoodie.common.table.log.block.HoodieLogBlock.HeaderMetadataType; -import com.uber.hoodie.common.table.log.block.HoodieLogBlock.HoodieLogBlockType; -import com.uber.hoodie.common.util.FSUtils; -import com.uber.hoodie.common.util.Option; -import com.uber.hoodie.exception.CorruptedLogFileException; -import com.uber.hoodie.exception.HoodieIOException; -import com.uber.hoodie.exception.HoodieNotSupportedException; import java.io.EOFException; import java.io.IOException; import java.util.Arrays; @@ -42,6 +29,19 @@ import org.apache.hadoop.fs.BufferedFSInputStream; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FSInputStream; import org.apache.hadoop.fs.FileSystem; +import org.apache.hudi.common.model.HoodieLogFile; +import org.apache.hudi.common.table.log.block.HoodieAvroDataBlock; +import org.apache.hudi.common.table.log.block.HoodieCommandBlock; +import org.apache.hudi.common.table.log.block.HoodieCorruptBlock; +import org.apache.hudi.common.table.log.block.HoodieDeleteBlock; +import org.apache.hudi.common.table.log.block.HoodieLogBlock; +import org.apache.hudi.common.table.log.block.HoodieLogBlock.HeaderMetadataType; +import org.apache.hudi.common.table.log.block.HoodieLogBlock.HoodieLogBlockType; +import org.apache.hudi.common.util.FSUtils; +import org.apache.hudi.common.util.Option; +import org.apache.hudi.exception.CorruptedLogFileException; +import org.apache.hudi.exception.HoodieIOException; +import org.apache.hudi.exception.HoodieNotSupportedException; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; diff --git a/hoodie-common/src/main/java/com/uber/hoodie/common/table/log/HoodieLogFormat.java b/hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieLogFormat.java similarity index 97% rename from hoodie-common/src/main/java/com/uber/hoodie/common/table/log/HoodieLogFormat.java rename to hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieLogFormat.java index 8bfbfc74e..aada3e2da 100644 --- a/hoodie-common/src/main/java/com/uber/hoodie/common/table/log/HoodieLogFormat.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieLogFormat.java @@ -16,19 +16,19 @@ * limitations under the License. */ -package com.uber.hoodie.common.table.log; +package org.apache.hudi.common.table.log; -import com.uber.hoodie.common.model.HoodieLogFile; -import com.uber.hoodie.common.table.log.block.HoodieLogBlock; -import com.uber.hoodie.common.util.FSUtils; -import com.uber.hoodie.common.util.Option; -import com.uber.hoodie.common.util.collection.Pair; import java.io.Closeable; import java.io.IOException; import java.util.Iterator; import org.apache.avro.Schema; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hudi.common.model.HoodieLogFile; +import org.apache.hudi.common.table.log.block.HoodieLogBlock; +import org.apache.hudi.common.util.FSUtils; +import org.apache.hudi.common.util.Option; +import org.apache.hudi.common.util.collection.Pair; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; diff --git a/hoodie-common/src/main/java/com/uber/hoodie/common/table/log/HoodieLogFormatReader.java b/hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieLogFormatReader.java similarity index 95% rename from hoodie-common/src/main/java/com/uber/hoodie/common/table/log/HoodieLogFormatReader.java rename to hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieLogFormatReader.java index 39ae5a933..fe935de88 100644 --- a/hoodie-common/src/main/java/com/uber/hoodie/common/table/log/HoodieLogFormatReader.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieLogFormatReader.java @@ -16,16 +16,16 @@ * limitations under the License. */ -package com.uber.hoodie.common.table.log; +package org.apache.hudi.common.table.log; -import com.uber.hoodie.common.model.HoodieLogFile; -import com.uber.hoodie.common.table.log.block.HoodieLogBlock; -import com.uber.hoodie.exception.HoodieIOException; import java.io.IOException; import java.util.ArrayList; import java.util.List; import org.apache.avro.Schema; import org.apache.hadoop.fs.FileSystem; +import org.apache.hudi.common.model.HoodieLogFile; +import org.apache.hudi.common.table.log.block.HoodieLogBlock; +import org.apache.hudi.exception.HoodieIOException; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; diff --git a/hoodie-common/src/main/java/com/uber/hoodie/common/table/log/HoodieLogFormatVersion.java b/hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieLogFormatVersion.java similarity index 98% rename from hoodie-common/src/main/java/com/uber/hoodie/common/table/log/HoodieLogFormatVersion.java rename to hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieLogFormatVersion.java index 4c105e10c..35a5f811f 100644 --- a/hoodie-common/src/main/java/com/uber/hoodie/common/table/log/HoodieLogFormatVersion.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieLogFormatVersion.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package com.uber.hoodie.common.table.log; +package org.apache.hudi.common.table.log; /** * Implements logic to determine behavior for feature flags for diff --git a/hoodie-common/src/main/java/com/uber/hoodie/common/table/log/HoodieLogFormatWriter.java b/hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieLogFormatWriter.java similarity index 95% rename from hoodie-common/src/main/java/com/uber/hoodie/common/table/log/HoodieLogFormatWriter.java rename to hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieLogFormatWriter.java index f7c3f08eb..2bf9b334e 100644 --- a/hoodie-common/src/main/java/com/uber/hoodie/common/table/log/HoodieLogFormatWriter.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieLogFormatWriter.java @@ -16,16 +16,8 @@ * limitations under the License. */ -package com.uber.hoodie.common.table.log; +package org.apache.hudi.common.table.log; -import com.uber.hoodie.common.model.HoodieLogFile; -import com.uber.hoodie.common.storage.StorageSchemes; -import com.uber.hoodie.common.table.log.HoodieLogFormat.Writer; -import com.uber.hoodie.common.table.log.HoodieLogFormat.WriterBuilder; -import com.uber.hoodie.common.table.log.block.HoodieLogBlock; -import com.uber.hoodie.common.util.FSUtils; -import com.uber.hoodie.exception.HoodieException; -import com.uber.hoodie.exception.HoodieIOException; import java.io.IOException; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileSystem; @@ -34,6 +26,14 @@ import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.hdfs.protocol.AlreadyBeingCreatedException; import org.apache.hadoop.hdfs.protocol.RecoveryInProgressException; import org.apache.hadoop.ipc.RemoteException; +import org.apache.hudi.common.model.HoodieLogFile; +import org.apache.hudi.common.storage.StorageSchemes; +import org.apache.hudi.common.table.log.HoodieLogFormat.Writer; +import org.apache.hudi.common.table.log.HoodieLogFormat.WriterBuilder; +import org.apache.hudi.common.table.log.block.HoodieLogBlock; +import org.apache.hudi.common.util.FSUtils; +import org.apache.hudi.exception.HoodieException; +import org.apache.hudi.exception.HoodieIOException; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; diff --git a/hoodie-common/src/main/java/com/uber/hoodie/common/table/log/HoodieMergedLogRecordScanner.java b/hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieMergedLogRecordScanner.java similarity index 91% rename from hoodie-common/src/main/java/com/uber/hoodie/common/table/log/HoodieMergedLogRecordScanner.java rename to hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieMergedLogRecordScanner.java index d99babe7b..b3361c368 100644 --- a/hoodie-common/src/main/java/com/uber/hoodie/common/table/log/HoodieMergedLogRecordScanner.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieMergedLogRecordScanner.java @@ -16,23 +16,23 @@ * limitations under the License. */ -package com.uber.hoodie.common.table.log; +package org.apache.hudi.common.table.log; -import com.uber.hoodie.common.model.HoodieKey; -import com.uber.hoodie.common.model.HoodieRecord; -import com.uber.hoodie.common.model.HoodieRecordPayload; -import com.uber.hoodie.common.util.DefaultSizeEstimator; -import com.uber.hoodie.common.util.HoodieRecordSizeEstimator; -import com.uber.hoodie.common.util.HoodieTimer; -import com.uber.hoodie.common.util.SpillableMapUtils; -import com.uber.hoodie.common.util.collection.ExternalSpillableMap; -import com.uber.hoodie.exception.HoodieIOException; import java.io.IOException; import java.util.Iterator; import java.util.List; import java.util.Map; import org.apache.avro.Schema; import org.apache.hadoop.fs.FileSystem; +import org.apache.hudi.common.model.HoodieKey; +import org.apache.hudi.common.model.HoodieRecord; +import org.apache.hudi.common.model.HoodieRecordPayload; +import org.apache.hudi.common.util.DefaultSizeEstimator; +import org.apache.hudi.common.util.HoodieRecordSizeEstimator; +import org.apache.hudi.common.util.HoodieTimer; +import org.apache.hudi.common.util.SpillableMapUtils; +import org.apache.hudi.common.util.collection.ExternalSpillableMap; +import org.apache.hudi.exception.HoodieIOException; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; diff --git a/hoodie-common/src/main/java/com/uber/hoodie/common/table/log/HoodieUnMergedLogRecordScanner.java b/hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieUnMergedLogRecordScanner.java similarity index 91% rename from hoodie-common/src/main/java/com/uber/hoodie/common/table/log/HoodieUnMergedLogRecordScanner.java rename to hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieUnMergedLogRecordScanner.java index 3a7984b9d..894db45dc 100644 --- a/hoodie-common/src/main/java/com/uber/hoodie/common/table/log/HoodieUnMergedLogRecordScanner.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieUnMergedLogRecordScanner.java @@ -16,14 +16,14 @@ * limitations under the License. */ -package com.uber.hoodie.common.table.log; +package org.apache.hudi.common.table.log; -import com.uber.hoodie.common.model.HoodieKey; -import com.uber.hoodie.common.model.HoodieRecord; -import com.uber.hoodie.common.model.HoodieRecordPayload; import java.util.List; import org.apache.avro.Schema; import org.apache.hadoop.fs.FileSystem; +import org.apache.hudi.common.model.HoodieKey; +import org.apache.hudi.common.model.HoodieRecord; +import org.apache.hudi.common.model.HoodieRecordPayload; public class HoodieUnMergedLogRecordScanner extends AbstractHoodieLogRecordScanner { diff --git a/hoodie-common/src/main/java/com/uber/hoodie/common/table/log/block/HoodieAvroDataBlock.java b/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieAvroDataBlock.java similarity index 97% rename from hoodie-common/src/main/java/com/uber/hoodie/common/table/log/block/HoodieAvroDataBlock.java rename to hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieAvroDataBlock.java index edc6f9d61..bc402e438 100644 --- a/hoodie-common/src/main/java/com/uber/hoodie/common/table/log/block/HoodieAvroDataBlock.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieAvroDataBlock.java @@ -16,14 +16,9 @@ * limitations under the License. */ -package com.uber.hoodie.common.table.log.block; +package org.apache.hudi.common.table.log.block; import com.google.common.annotations.VisibleForTesting; -import com.uber.hoodie.common.model.HoodieLogFile; -import com.uber.hoodie.common.storage.SizeAwareDataInputStream; -import com.uber.hoodie.common.util.HoodieAvroUtils; -import com.uber.hoodie.common.util.Option; -import com.uber.hoodie.exception.HoodieIOException; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.DataInputStream; @@ -46,6 +41,11 @@ import org.apache.avro.io.DecoderFactory; import org.apache.avro.io.Encoder; import org.apache.avro.io.EncoderFactory; import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hudi.common.model.HoodieLogFile; +import org.apache.hudi.common.storage.SizeAwareDataInputStream; +import org.apache.hudi.common.util.HoodieAvroUtils; +import org.apache.hudi.common.util.Option; +import org.apache.hudi.exception.HoodieIOException; /** * DataBlock contains a list of records serialized using Avro. The Datablock contains 1. Data Block diff --git a/hoodie-common/src/main/java/com/uber/hoodie/common/table/log/block/HoodieAvroDataBlockVersion.java b/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieAvroDataBlockVersion.java similarity index 96% rename from hoodie-common/src/main/java/com/uber/hoodie/common/table/log/block/HoodieAvroDataBlockVersion.java rename to hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieAvroDataBlockVersion.java index dcd438594..84f88f668 100644 --- a/hoodie-common/src/main/java/com/uber/hoodie/common/table/log/block/HoodieAvroDataBlockVersion.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieAvroDataBlockVersion.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package com.uber.hoodie.common.table.log.block; +package org.apache.hudi.common.table.log.block; /** * A set of feature flags associated with a data log block format. Versions are changed when the log diff --git a/hoodie-common/src/main/java/com/uber/hoodie/common/table/log/block/HoodieCommandBlock.java b/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieCommandBlock.java similarity index 94% rename from hoodie-common/src/main/java/com/uber/hoodie/common/table/log/block/HoodieCommandBlock.java rename to hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieCommandBlock.java index acb25ee4e..dc4ff6622 100644 --- a/hoodie-common/src/main/java/com/uber/hoodie/common/table/log/block/HoodieCommandBlock.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieCommandBlock.java @@ -16,13 +16,13 @@ * limitations under the License. */ -package com.uber.hoodie.common.table.log.block; +package org.apache.hudi.common.table.log.block; -import com.uber.hoodie.common.model.HoodieLogFile; -import com.uber.hoodie.common.util.Option; import java.util.HashMap; import java.util.Map; import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hudi.common.model.HoodieLogFile; +import org.apache.hudi.common.util.Option; /** * Command block issues a specific command to the scanner diff --git a/hoodie-common/src/main/java/com/uber/hoodie/common/table/log/block/HoodieCommandBlockVersion.java b/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieCommandBlockVersion.java similarity index 95% rename from hoodie-common/src/main/java/com/uber/hoodie/common/table/log/block/HoodieCommandBlockVersion.java rename to hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieCommandBlockVersion.java index e89844901..4ba7db646 100644 --- a/hoodie-common/src/main/java/com/uber/hoodie/common/table/log/block/HoodieCommandBlockVersion.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieCommandBlockVersion.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package com.uber.hoodie.common.table.log.block; +package org.apache.hudi.common.table.log.block; /** * A set of feature flags associated with a command log block format. Versions are changed when the diff --git a/hoodie-common/src/main/java/com/uber/hoodie/common/table/log/block/HoodieCorruptBlock.java b/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieCorruptBlock.java similarity index 94% rename from hoodie-common/src/main/java/com/uber/hoodie/common/table/log/block/HoodieCorruptBlock.java rename to hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieCorruptBlock.java index 7dd4cdbe9..fb26c6f24 100644 --- a/hoodie-common/src/main/java/com/uber/hoodie/common/table/log/block/HoodieCorruptBlock.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieCorruptBlock.java @@ -16,13 +16,13 @@ * limitations under the License. */ -package com.uber.hoodie.common.table.log.block; +package org.apache.hudi.common.table.log.block; -import com.uber.hoodie.common.model.HoodieLogFile; -import com.uber.hoodie.common.util.Option; import java.io.IOException; import java.util.Map; import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hudi.common.model.HoodieLogFile; +import org.apache.hudi.common.util.Option; /** * Corrupt block is emitted whenever the scanner finds the length of the block written at the diff --git a/hoodie-common/src/main/java/com/uber/hoodie/common/table/log/block/HoodieDeleteBlock.java b/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieDeleteBlock.java similarity index 90% rename from hoodie-common/src/main/java/com/uber/hoodie/common/table/log/block/HoodieDeleteBlock.java rename to hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieDeleteBlock.java index a2b695351..71942aa25 100644 --- a/hoodie-common/src/main/java/com/uber/hoodie/common/table/log/block/HoodieDeleteBlock.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieDeleteBlock.java @@ -16,14 +16,8 @@ * limitations under the License. */ -package com.uber.hoodie.common.table.log.block; +package org.apache.hudi.common.table.log.block; -import com.uber.hoodie.common.model.HoodieKey; -import com.uber.hoodie.common.model.HoodieLogFile; -import com.uber.hoodie.common.storage.SizeAwareDataInputStream; -import com.uber.hoodie.common.util.Option; -import com.uber.hoodie.common.util.SerializationUtils; -import com.uber.hoodie.exception.HoodieIOException; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.DataInputStream; @@ -32,6 +26,12 @@ import java.io.IOException; import java.util.HashMap; import java.util.Map; import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hudi.common.model.HoodieKey; +import org.apache.hudi.common.model.HoodieLogFile; +import org.apache.hudi.common.storage.SizeAwareDataInputStream; +import org.apache.hudi.common.util.Option; +import org.apache.hudi.common.util.SerializationUtils; +import org.apache.hudi.exception.HoodieIOException; /** * Delete block contains a list of keys to be deleted from scanning the blocks so far @@ -67,7 +67,7 @@ public class HoodieDeleteBlock extends HoodieLogBlock { ByteArrayOutputStream baos = new ByteArrayOutputStream(); DataOutputStream output = new DataOutputStream(baos); byte[] bytesToWrite = SerializationUtils.serialize(getKeysToDelete()); - output.writeInt(HoodieLogBlock.version); + output.writeInt(version); output.writeInt(bytesToWrite.length); output.write(bytesToWrite); return baos.toByteArray(); diff --git a/hoodie-common/src/main/java/com/uber/hoodie/common/table/log/block/HoodieDeleteBlockVersion.java b/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieDeleteBlockVersion.java similarity index 95% rename from hoodie-common/src/main/java/com/uber/hoodie/common/table/log/block/HoodieDeleteBlockVersion.java rename to hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieDeleteBlockVersion.java index e4f6edf24..6fa9e7f40 100644 --- a/hoodie-common/src/main/java/com/uber/hoodie/common/table/log/block/HoodieDeleteBlockVersion.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieDeleteBlockVersion.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package com.uber.hoodie.common.table.log.block; +package org.apache.hudi.common.table.log.block; /** * A set of feature flags associated with a delete log block format. Versions are changed when the diff --git a/hoodie-common/src/main/java/com/uber/hoodie/common/table/log/block/HoodieLogBlock.java b/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieLogBlock.java similarity index 96% rename from hoodie-common/src/main/java/com/uber/hoodie/common/table/log/block/HoodieLogBlock.java rename to hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieLogBlock.java index d3ad4c258..c558a032f 100644 --- a/hoodie-common/src/main/java/com/uber/hoodie/common/table/log/block/HoodieLogBlock.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieLogBlock.java @@ -16,15 +16,9 @@ * limitations under the License. */ -package com.uber.hoodie.common.table.log.block; +package org.apache.hudi.common.table.log.block; import com.google.common.collect.Maps; -import com.uber.hoodie.common.model.HoodieLogFile; -import com.uber.hoodie.common.table.log.HoodieMergedLogRecordScanner; -import com.uber.hoodie.common.util.FSUtils; -import com.uber.hoodie.common.util.Option; -import com.uber.hoodie.exception.HoodieException; -import com.uber.hoodie.exception.HoodieIOException; import java.io.ByteArrayOutputStream; import java.io.DataInputStream; import java.io.DataOutputStream; @@ -33,6 +27,12 @@ import java.io.IOException; import java.util.Map; import javax.annotation.Nonnull; import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hudi.common.model.HoodieLogFile; +import org.apache.hudi.common.table.log.HoodieMergedLogRecordScanner; +import org.apache.hudi.common.util.FSUtils; +import org.apache.hudi.common.util.Option; +import org.apache.hudi.exception.HoodieException; +import org.apache.hudi.exception.HoodieIOException; /** * Abstract class defining a block in HoodieLogFile diff --git a/hoodie-common/src/main/java/com/uber/hoodie/common/table/log/block/HoodieLogBlockVersion.java b/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieLogBlockVersion.java similarity index 95% rename from hoodie-common/src/main/java/com/uber/hoodie/common/table/log/block/HoodieLogBlockVersion.java rename to hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieLogBlockVersion.java index 1beb4b0b0..440dc2204 100644 --- a/hoodie-common/src/main/java/com/uber/hoodie/common/table/log/block/HoodieLogBlockVersion.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieLogBlockVersion.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package com.uber.hoodie.common.table.log.block; +package org.apache.hudi.common.table.log.block; abstract class HoodieLogBlockVersion { diff --git a/hoodie-common/src/main/java/com/uber/hoodie/common/table/timeline/HoodieActiveTimeline.java b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/HoodieActiveTimeline.java similarity index 98% rename from hoodie-common/src/main/java/com/uber/hoodie/common/table/timeline/HoodieActiveTimeline.java rename to hudi-common/src/main/java/org/apache/hudi/common/table/timeline/HoodieActiveTimeline.java index 0509e5b67..c86c3a111 100644 --- a/hoodie-common/src/main/java/com/uber/hoodie/common/table/timeline/HoodieActiveTimeline.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/HoodieActiveTimeline.java @@ -16,16 +16,11 @@ * limitations under the License. */ -package com.uber.hoodie.common.table.timeline; +package org.apache.hudi.common.table.timeline; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Sets; -import com.uber.hoodie.common.table.HoodieTableMetaClient; -import com.uber.hoodie.common.table.HoodieTimeline; -import com.uber.hoodie.common.table.timeline.HoodieInstant.State; -import com.uber.hoodie.common.util.Option; -import com.uber.hoodie.exception.HoodieIOException; import java.io.IOException; import java.io.Serializable; import java.text.SimpleDateFormat; @@ -39,6 +34,11 @@ import org.apache.commons.io.IOUtils; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.Path; +import org.apache.hudi.common.table.HoodieTableMetaClient; +import org.apache.hudi.common.table.HoodieTimeline; +import org.apache.hudi.common.table.timeline.HoodieInstant.State; +import org.apache.hudi.common.util.Option; +import org.apache.hudi.exception.HoodieIOException; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; diff --git a/hoodie-common/src/main/java/com/uber/hoodie/common/table/timeline/HoodieArchivedTimeline.java b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/HoodieArchivedTimeline.java similarity index 94% rename from hoodie-common/src/main/java/com/uber/hoodie/common/table/timeline/HoodieArchivedTimeline.java rename to hudi-common/src/main/java/org/apache/hudi/common/table/timeline/HoodieArchivedTimeline.java index d00e79db4..105b98271 100644 --- a/hoodie-common/src/main/java/com/uber/hoodie/common/table/timeline/HoodieArchivedTimeline.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/HoodieArchivedTimeline.java @@ -16,12 +16,8 @@ * limitations under the License. */ -package com.uber.hoodie.common.table.timeline; +package org.apache.hudi.common.table.timeline; -import com.uber.hoodie.common.table.HoodieTableMetaClient; -import com.uber.hoodie.common.table.HoodieTimeline; -import com.uber.hoodie.common.util.Option; -import com.uber.hoodie.exception.HoodieIOException; import java.io.IOException; import java.io.Serializable; import java.util.Arrays; @@ -32,6 +28,10 @@ import java.util.stream.Collectors; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.SequenceFile; import org.apache.hadoop.io.Text; +import org.apache.hudi.common.table.HoodieTableMetaClient; +import org.apache.hudi.common.table.HoodieTimeline; +import org.apache.hudi.common.util.Option; +import org.apache.hudi.exception.HoodieIOException; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; diff --git a/hoodie-common/src/main/java/com/uber/hoodie/common/table/timeline/HoodieDefaultTimeline.java b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/HoodieDefaultTimeline.java similarity index 96% rename from hoodie-common/src/main/java/com/uber/hoodie/common/table/timeline/HoodieDefaultTimeline.java rename to hudi-common/src/main/java/org/apache/hudi/common/table/timeline/HoodieDefaultTimeline.java index aac13f1b8..9f8154f87 100644 --- a/hoodie-common/src/main/java/com/uber/hoodie/common/table/timeline/HoodieDefaultTimeline.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/HoodieDefaultTimeline.java @@ -16,13 +16,9 @@ * limitations under the License. */ -package com.uber.hoodie.common.table.timeline; +package org.apache.hudi.common.table.timeline; import com.google.common.collect.Sets; -import com.uber.hoodie.common.table.HoodieTimeline; -import com.uber.hoodie.common.util.Option; -import com.uber.hoodie.common.util.StringUtils; -import com.uber.hoodie.exception.HoodieException; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; import java.util.List; @@ -32,6 +28,10 @@ import java.util.function.Predicate; import java.util.stream.Collectors; import java.util.stream.Stream; import org.apache.commons.codec.binary.Hex; +import org.apache.hudi.common.table.HoodieTimeline; +import org.apache.hudi.common.util.Option; +import org.apache.hudi.common.util.StringUtils; +import org.apache.hudi.exception.HoodieException; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; diff --git a/hoodie-common/src/main/java/com/uber/hoodie/common/table/timeline/HoodieInstant.java b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/HoodieInstant.java similarity index 97% rename from hoodie-common/src/main/java/com/uber/hoodie/common/table/timeline/HoodieInstant.java rename to hudi-common/src/main/java/org/apache/hudi/common/table/timeline/HoodieInstant.java index 0d50787c0..33fb8bd32 100644 --- a/hoodie-common/src/main/java/com/uber/hoodie/common/table/timeline/HoodieInstant.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/HoodieInstant.java @@ -16,13 +16,13 @@ * limitations under the License. */ -package com.uber.hoodie.common.table.timeline; +package org.apache.hudi.common.table.timeline; -import com.uber.hoodie.common.table.HoodieTimeline; -import com.uber.hoodie.common.util.FSUtils; import java.io.Serializable; import java.util.Objects; import org.apache.hadoop.fs.FileStatus; +import org.apache.hudi.common.table.HoodieTimeline; +import org.apache.hudi.common.util.FSUtils; /** * A Hoodie Instant represents a action done on a hoodie dataset. All actions start with a inflight diff --git a/hoodie-common/src/main/java/com/uber/hoodie/common/table/timeline/dto/CompactionOpDTO.java b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/dto/CompactionOpDTO.java similarity index 92% rename from hoodie-common/src/main/java/com/uber/hoodie/common/table/timeline/dto/CompactionOpDTO.java rename to hudi-common/src/main/java/org/apache/hudi/common/table/timeline/dto/CompactionOpDTO.java index 22a953e8b..386c303b6 100644 --- a/hoodie-common/src/main/java/com/uber/hoodie/common/table/timeline/dto/CompactionOpDTO.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/dto/CompactionOpDTO.java @@ -16,17 +16,17 @@ * limitations under the License. */ -package com.uber.hoodie.common.table.timeline.dto; +package org.apache.hudi.common.table.timeline.dto; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonProperty; -import com.uber.hoodie.common.model.CompactionOperation; -import com.uber.hoodie.common.util.Option; -import com.uber.hoodie.common.util.collection.Pair; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; +import org.apache.hudi.common.model.CompactionOperation; +import org.apache.hudi.common.util.Option; +import org.apache.hudi.common.util.collection.Pair; @JsonIgnoreProperties(ignoreUnknown = true) public class CompactionOpDTO { diff --git a/hoodie-common/src/main/java/com/uber/hoodie/common/table/timeline/dto/DataFileDTO.java b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/dto/DataFileDTO.java similarity index 94% rename from hoodie-common/src/main/java/com/uber/hoodie/common/table/timeline/dto/DataFileDTO.java rename to hudi-common/src/main/java/org/apache/hudi/common/table/timeline/dto/DataFileDTO.java index 2415d139a..cb587147a 100644 --- a/hoodie-common/src/main/java/com/uber/hoodie/common/table/timeline/dto/DataFileDTO.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/dto/DataFileDTO.java @@ -16,11 +16,11 @@ * limitations under the License. */ -package com.uber.hoodie.common.table.timeline.dto; +package org.apache.hudi.common.table.timeline.dto; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonProperty; -import com.uber.hoodie.common.model.HoodieDataFile; +import org.apache.hudi.common.model.HoodieDataFile; @JsonIgnoreProperties(ignoreUnknown = true) public class DataFileDTO { diff --git a/hoodie-common/src/main/java/com/uber/hoodie/common/table/timeline/dto/FSPermissionDTO.java b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/dto/FSPermissionDTO.java similarity index 97% rename from hoodie-common/src/main/java/com/uber/hoodie/common/table/timeline/dto/FSPermissionDTO.java rename to hudi-common/src/main/java/org/apache/hudi/common/table/timeline/dto/FSPermissionDTO.java index 86de008a3..632309704 100644 --- a/hoodie-common/src/main/java/com/uber/hoodie/common/table/timeline/dto/FSPermissionDTO.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/dto/FSPermissionDTO.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package com.uber.hoodie.common.table.timeline.dto; +package org.apache.hudi.common.table.timeline.dto; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonProperty; diff --git a/hoodie-common/src/main/java/com/uber/hoodie/common/table/timeline/dto/FileGroupDTO.java b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/dto/FileGroupDTO.java similarity index 92% rename from hoodie-common/src/main/java/com/uber/hoodie/common/table/timeline/dto/FileGroupDTO.java rename to hudi-common/src/main/java/org/apache/hudi/common/table/timeline/dto/FileGroupDTO.java index 484239807..6699ed552 100644 --- a/hoodie-common/src/main/java/com/uber/hoodie/common/table/timeline/dto/FileGroupDTO.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/dto/FileGroupDTO.java @@ -16,14 +16,14 @@ * limitations under the License. */ -package com.uber.hoodie.common.table.timeline.dto; +package org.apache.hudi.common.table.timeline.dto; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonProperty; -import com.uber.hoodie.common.model.HoodieFileGroup; -import com.uber.hoodie.common.table.HoodieTableMetaClient; import java.util.List; import java.util.stream.Collectors; +import org.apache.hudi.common.model.HoodieFileGroup; +import org.apache.hudi.common.table.HoodieTableMetaClient; @JsonIgnoreProperties(ignoreUnknown = true) public class FileGroupDTO { diff --git a/hoodie-common/src/main/java/com/uber/hoodie/common/table/timeline/dto/FilePathDTO.java b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/dto/FilePathDTO.java similarity index 96% rename from hoodie-common/src/main/java/com/uber/hoodie/common/table/timeline/dto/FilePathDTO.java rename to hudi-common/src/main/java/org/apache/hudi/common/table/timeline/dto/FilePathDTO.java index 2450b52ac..eae4f4aa2 100644 --- a/hoodie-common/src/main/java/com/uber/hoodie/common/table/timeline/dto/FilePathDTO.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/dto/FilePathDTO.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package com.uber.hoodie.common.table.timeline.dto; +package org.apache.hudi.common.table.timeline.dto; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonProperty; diff --git a/hoodie-common/src/main/java/com/uber/hoodie/common/table/timeline/dto/FileSliceDTO.java b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/dto/FileSliceDTO.java similarity index 95% rename from hoodie-common/src/main/java/com/uber/hoodie/common/table/timeline/dto/FileSliceDTO.java rename to hudi-common/src/main/java/org/apache/hudi/common/table/timeline/dto/FileSliceDTO.java index 12d914fd5..5bfd3fbc3 100644 --- a/hoodie-common/src/main/java/com/uber/hoodie/common/table/timeline/dto/FileSliceDTO.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/dto/FileSliceDTO.java @@ -16,13 +16,13 @@ * limitations under the License. */ -package com.uber.hoodie.common.table.timeline.dto; +package org.apache.hudi.common.table.timeline.dto; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonProperty; -import com.uber.hoodie.common.model.FileSlice; import java.util.List; import java.util.stream.Collectors; +import org.apache.hudi.common.model.FileSlice; @JsonIgnoreProperties(ignoreUnknown = true) public class FileSliceDTO { diff --git a/hoodie-common/src/main/java/com/uber/hoodie/common/table/timeline/dto/FileStatusDTO.java b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/dto/FileStatusDTO.java similarity index 97% rename from hoodie-common/src/main/java/com/uber/hoodie/common/table/timeline/dto/FileStatusDTO.java rename to hudi-common/src/main/java/org/apache/hudi/common/table/timeline/dto/FileStatusDTO.java index 897312e66..79fa35c9a 100644 --- a/hoodie-common/src/main/java/com/uber/hoodie/common/table/timeline/dto/FileStatusDTO.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/dto/FileStatusDTO.java @@ -16,13 +16,13 @@ * limitations under the License. */ -package com.uber.hoodie.common.table.timeline.dto; +package org.apache.hudi.common.table.timeline.dto; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonProperty; -import com.uber.hoodie.exception.HoodieException; import java.io.IOException; import org.apache.hadoop.fs.FileStatus; +import org.apache.hudi.exception.HoodieException; @JsonIgnoreProperties(ignoreUnknown = true) public class FileStatusDTO { diff --git a/hoodie-common/src/main/java/com/uber/hoodie/common/table/timeline/dto/InstantDTO.java b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/dto/InstantDTO.java similarity index 93% rename from hoodie-common/src/main/java/com/uber/hoodie/common/table/timeline/dto/InstantDTO.java rename to hudi-common/src/main/java/org/apache/hudi/common/table/timeline/dto/InstantDTO.java index e27b7cce2..4d51fd628 100644 --- a/hoodie-common/src/main/java/com/uber/hoodie/common/table/timeline/dto/InstantDTO.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/dto/InstantDTO.java @@ -16,11 +16,11 @@ * limitations under the License. */ -package com.uber.hoodie.common.table.timeline.dto; +package org.apache.hudi.common.table.timeline.dto; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonProperty; -import com.uber.hoodie.common.table.timeline.HoodieInstant; +import org.apache.hudi.common.table.timeline.HoodieInstant; @JsonIgnoreProperties(ignoreUnknown = true) diff --git a/hoodie-common/src/main/java/com/uber/hoodie/common/table/timeline/dto/LogFileDTO.java b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/dto/LogFileDTO.java similarity index 94% rename from hoodie-common/src/main/java/com/uber/hoodie/common/table/timeline/dto/LogFileDTO.java rename to hudi-common/src/main/java/org/apache/hudi/common/table/timeline/dto/LogFileDTO.java index b3ea3e038..bb3345b46 100644 --- a/hoodie-common/src/main/java/com/uber/hoodie/common/table/timeline/dto/LogFileDTO.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/dto/LogFileDTO.java @@ -16,12 +16,12 @@ * limitations under the License. */ -package com.uber.hoodie.common.table.timeline.dto; +package org.apache.hudi.common.table.timeline.dto; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonProperty; -import com.uber.hoodie.common.model.HoodieLogFile; import org.apache.hadoop.fs.FileStatus; +import org.apache.hudi.common.model.HoodieLogFile; @JsonIgnoreProperties(ignoreUnknown = true) public class LogFileDTO { diff --git a/hoodie-common/src/main/java/com/uber/hoodie/common/table/timeline/dto/TimelineDTO.java b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/dto/TimelineDTO.java similarity index 88% rename from hoodie-common/src/main/java/com/uber/hoodie/common/table/timeline/dto/TimelineDTO.java rename to hudi-common/src/main/java/org/apache/hudi/common/table/timeline/dto/TimelineDTO.java index 60a740223..f31a57092 100644 --- a/hoodie-common/src/main/java/com/uber/hoodie/common/table/timeline/dto/TimelineDTO.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/dto/TimelineDTO.java @@ -16,15 +16,15 @@ * limitations under the License. */ -package com.uber.hoodie.common.table.timeline.dto; +package org.apache.hudi.common.table.timeline.dto; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonProperty; -import com.uber.hoodie.common.table.HoodieTableMetaClient; -import com.uber.hoodie.common.table.HoodieTimeline; -import com.uber.hoodie.common.table.timeline.HoodieDefaultTimeline; import java.util.List; import java.util.stream.Collectors; +import org.apache.hudi.common.table.HoodieTableMetaClient; +import org.apache.hudi.common.table.HoodieTimeline; +import org.apache.hudi.common.table.timeline.HoodieDefaultTimeline; @JsonIgnoreProperties(ignoreUnknown = true) public class TimelineDTO { diff --git a/hoodie-common/src/main/java/com/uber/hoodie/common/table/view/AbstractTableFileSystemView.java b/hudi-common/src/main/java/org/apache/hudi/common/table/view/AbstractTableFileSystemView.java similarity index 97% rename from hoodie-common/src/main/java/com/uber/hoodie/common/table/view/AbstractTableFileSystemView.java rename to hudi-common/src/main/java/org/apache/hudi/common/table/view/AbstractTableFileSystemView.java index 0a2863989..6bd98027b 100644 --- a/hoodie-common/src/main/java/com/uber/hoodie/common/table/view/AbstractTableFileSystemView.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/view/AbstractTableFileSystemView.java @@ -16,25 +16,9 @@ * limitations under the License. */ -package com.uber.hoodie.common.table.view; +package org.apache.hudi.common.table.view; import com.google.common.base.Preconditions; -import com.uber.hoodie.common.model.CompactionOperation; -import com.uber.hoodie.common.model.FileSlice; -import com.uber.hoodie.common.model.HoodieDataFile; -import com.uber.hoodie.common.model.HoodieFileGroup; -import com.uber.hoodie.common.model.HoodieFileGroupId; -import com.uber.hoodie.common.model.HoodieLogFile; -import com.uber.hoodie.common.table.HoodieTableMetaClient; -import com.uber.hoodie.common.table.HoodieTimeline; -import com.uber.hoodie.common.table.SyncableFileSystemView; -import com.uber.hoodie.common.table.timeline.HoodieInstant; -import com.uber.hoodie.common.util.CompactionUtils; -import com.uber.hoodie.common.util.FSUtils; -import com.uber.hoodie.common.util.HoodieTimer; -import com.uber.hoodie.common.util.Option; -import com.uber.hoodie.common.util.collection.Pair; -import com.uber.hoodie.exception.HoodieIOException; import java.io.IOException; import java.io.Serializable; import java.util.ArrayList; @@ -52,6 +36,22 @@ import java.util.stream.Collectors; import java.util.stream.Stream; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.Path; +import org.apache.hudi.common.model.CompactionOperation; +import org.apache.hudi.common.model.FileSlice; +import org.apache.hudi.common.model.HoodieDataFile; +import org.apache.hudi.common.model.HoodieFileGroup; +import org.apache.hudi.common.model.HoodieFileGroupId; +import org.apache.hudi.common.model.HoodieLogFile; +import org.apache.hudi.common.table.HoodieTableMetaClient; +import org.apache.hudi.common.table.HoodieTimeline; +import org.apache.hudi.common.table.SyncableFileSystemView; +import org.apache.hudi.common.table.timeline.HoodieInstant; +import org.apache.hudi.common.util.CompactionUtils; +import org.apache.hudi.common.util.FSUtils; +import org.apache.hudi.common.util.HoodieTimer; +import org.apache.hudi.common.util.Option; +import org.apache.hudi.common.util.collection.Pair; +import org.apache.hudi.exception.HoodieIOException; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; diff --git a/hoodie-common/src/main/java/com/uber/hoodie/common/table/view/FileSystemViewManager.java b/hudi-common/src/main/java/org/apache/hudi/common/table/view/FileSystemViewManager.java similarity index 96% rename from hoodie-common/src/main/java/com/uber/hoodie/common/table/view/FileSystemViewManager.java rename to hudi-common/src/main/java/org/apache/hudi/common/table/view/FileSystemViewManager.java index 9086a91fc..c82e33a46 100644 --- a/hoodie-common/src/main/java/com/uber/hoodie/common/table/view/FileSystemViewManager.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/view/FileSystemViewManager.java @@ -16,14 +16,14 @@ * limitations under the License. */ -package com.uber.hoodie.common.table.view; +package org.apache.hudi.common.table.view; -import com.uber.hoodie.common.SerializableConfiguration; -import com.uber.hoodie.common.table.HoodieTableMetaClient; -import com.uber.hoodie.common.table.HoodieTimeline; -import com.uber.hoodie.common.table.SyncableFileSystemView; -import com.uber.hoodie.common.util.Functions.Function2; import java.util.concurrent.ConcurrentHashMap; +import org.apache.hudi.common.SerializableConfiguration; +import org.apache.hudi.common.table.HoodieTableMetaClient; +import org.apache.hudi.common.table.HoodieTimeline; +import org.apache.hudi.common.table.SyncableFileSystemView; +import org.apache.hudi.common.util.Functions.Function2; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; diff --git a/hoodie-common/src/main/java/com/uber/hoodie/common/table/view/FileSystemViewStorageConfig.java b/hudi-common/src/main/java/org/apache/hudi/common/table/view/FileSystemViewStorageConfig.java similarity index 98% rename from hoodie-common/src/main/java/com/uber/hoodie/common/table/view/FileSystemViewStorageConfig.java rename to hudi-common/src/main/java/org/apache/hudi/common/table/view/FileSystemViewStorageConfig.java index fe79ebf53..3924b8b26 100644 --- a/hoodie-common/src/main/java/com/uber/hoodie/common/table/view/FileSystemViewStorageConfig.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/view/FileSystemViewStorageConfig.java @@ -16,14 +16,14 @@ * limitations under the License. */ -package com.uber.hoodie.common.table.view; +package org.apache.hudi.common.table.view; import com.google.common.base.Preconditions; -import com.uber.hoodie.config.DefaultHoodieConfig; import java.io.File; import java.io.FileReader; import java.io.IOException; import java.util.Properties; +import org.apache.hudi.config.DefaultHoodieConfig; /** * File System View Storage Configurations diff --git a/hoodie-common/src/main/java/com/uber/hoodie/common/table/view/FileSystemViewStorageType.java b/hudi-common/src/main/java/org/apache/hudi/common/table/view/FileSystemViewStorageType.java similarity index 97% rename from hoodie-common/src/main/java/com/uber/hoodie/common/table/view/FileSystemViewStorageType.java rename to hudi-common/src/main/java/org/apache/hudi/common/table/view/FileSystemViewStorageType.java index aa2394958..4ccd69f0d 100644 --- a/hoodie-common/src/main/java/com/uber/hoodie/common/table/view/FileSystemViewStorageType.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/view/FileSystemViewStorageType.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package com.uber.hoodie.common.table.view; +package org.apache.hudi.common.table.view; /** * Storage Type used to store/retrieve File system view of a table diff --git a/hoodie-common/src/main/java/com/uber/hoodie/common/table/view/HoodieTableFileSystemView.java b/hudi-common/src/main/java/org/apache/hudi/common/table/view/HoodieTableFileSystemView.java similarity index 94% rename from hoodie-common/src/main/java/com/uber/hoodie/common/table/view/HoodieTableFileSystemView.java rename to hudi-common/src/main/java/org/apache/hudi/common/table/view/HoodieTableFileSystemView.java index 741d7eee2..5b9601e6e 100644 --- a/hoodie-common/src/main/java/com/uber/hoodie/common/table/view/HoodieTableFileSystemView.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/view/HoodieTableFileSystemView.java @@ -16,17 +16,9 @@ * limitations under the License. */ -package com.uber.hoodie.common.table.view; +package org.apache.hudi.common.table.view; import com.google.common.base.Preconditions; -import com.uber.hoodie.common.model.CompactionOperation; -import com.uber.hoodie.common.model.HoodieFileGroup; -import com.uber.hoodie.common.model.HoodieFileGroupId; -import com.uber.hoodie.common.table.HoodieTableMetaClient; -import com.uber.hoodie.common.table.HoodieTimeline; -import com.uber.hoodie.common.table.TableFileSystemView; -import com.uber.hoodie.common.util.Option; -import com.uber.hoodie.common.util.collection.Pair; import java.io.IOException; import java.util.ArrayList; import java.util.List; @@ -35,6 +27,14 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.stream.Collectors; import java.util.stream.Stream; import org.apache.hadoop.fs.FileStatus; +import org.apache.hudi.common.model.CompactionOperation; +import org.apache.hudi.common.model.HoodieFileGroup; +import org.apache.hudi.common.model.HoodieFileGroupId; +import org.apache.hudi.common.table.HoodieTableMetaClient; +import org.apache.hudi.common.table.HoodieTimeline; +import org.apache.hudi.common.table.TableFileSystemView; +import org.apache.hudi.common.util.Option; +import org.apache.hudi.common.util.collection.Pair; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; diff --git a/hoodie-common/src/main/java/com/uber/hoodie/common/table/view/IncrementalTimelineSyncFileSystemView.java b/hudi-common/src/main/java/org/apache/hudi/common/table/view/IncrementalTimelineSyncFileSystemView.java similarity index 93% rename from hoodie-common/src/main/java/com/uber/hoodie/common/table/view/IncrementalTimelineSyncFileSystemView.java rename to hudi-common/src/main/java/org/apache/hudi/common/table/view/IncrementalTimelineSyncFileSystemView.java index e6ccf59b7..22cc53bbd 100644 --- a/hoodie-common/src/main/java/com/uber/hoodie/common/table/view/IncrementalTimelineSyncFileSystemView.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/view/IncrementalTimelineSyncFileSystemView.java @@ -16,33 +16,33 @@ * limitations under the License. */ -package com.uber.hoodie.common.table.view; +package org.apache.hudi.common.table.view; -import com.uber.hoodie.avro.model.HoodieCleanMetadata; -import com.uber.hoodie.avro.model.HoodieCompactionPlan; -import com.uber.hoodie.avro.model.HoodieRestoreMetadata; -import com.uber.hoodie.avro.model.HoodieRollbackMetadata; -import com.uber.hoodie.common.model.CompactionOperation; -import com.uber.hoodie.common.model.FileSlice; -import com.uber.hoodie.common.model.HoodieCommitMetadata; -import com.uber.hoodie.common.model.HoodieDataFile; -import com.uber.hoodie.common.model.HoodieFileGroup; -import com.uber.hoodie.common.model.HoodieLogFile; -import com.uber.hoodie.common.table.HoodieTimeline; -import com.uber.hoodie.common.table.timeline.HoodieInstant; -import com.uber.hoodie.common.util.AvroUtils; -import com.uber.hoodie.common.util.CompactionUtils; -import com.uber.hoodie.common.util.Option; -import com.uber.hoodie.common.util.TimelineDiffHelper; -import com.uber.hoodie.common.util.TimelineDiffHelper.TimelineDiffResult; -import com.uber.hoodie.common.util.collection.Pair; -import com.uber.hoodie.exception.HoodieException; import java.io.IOException; import java.util.List; import java.util.Map; import java.util.stream.Collectors; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.Path; +import org.apache.hudi.avro.model.HoodieCleanMetadata; +import org.apache.hudi.avro.model.HoodieCompactionPlan; +import org.apache.hudi.avro.model.HoodieRestoreMetadata; +import org.apache.hudi.avro.model.HoodieRollbackMetadata; +import org.apache.hudi.common.model.CompactionOperation; +import org.apache.hudi.common.model.FileSlice; +import org.apache.hudi.common.model.HoodieCommitMetadata; +import org.apache.hudi.common.model.HoodieDataFile; +import org.apache.hudi.common.model.HoodieFileGroup; +import org.apache.hudi.common.model.HoodieLogFile; +import org.apache.hudi.common.table.HoodieTimeline; +import org.apache.hudi.common.table.timeline.HoodieInstant; +import org.apache.hudi.common.util.AvroUtils; +import org.apache.hudi.common.util.CompactionUtils; +import org.apache.hudi.common.util.Option; +import org.apache.hudi.common.util.TimelineDiffHelper; +import org.apache.hudi.common.util.TimelineDiffHelper.TimelineDiffResult; +import org.apache.hudi.common.util.collection.Pair; +import org.apache.hudi.exception.HoodieException; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; diff --git a/hoodie-common/src/main/java/com/uber/hoodie/common/table/view/PriorityBasedFileSystemView.java b/hudi-common/src/main/java/org/apache/hudi/common/table/view/PriorityBasedFileSystemView.java similarity index 91% rename from hoodie-common/src/main/java/com/uber/hoodie/common/table/view/PriorityBasedFileSystemView.java rename to hudi-common/src/main/java/org/apache/hudi/common/table/view/PriorityBasedFileSystemView.java index 601c5f20f..2275de208 100644 --- a/hoodie-common/src/main/java/com/uber/hoodie/common/table/view/PriorityBasedFileSystemView.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/view/PriorityBasedFileSystemView.java @@ -16,24 +16,24 @@ * limitations under the License. */ -package com.uber.hoodie.common.table.view; +package org.apache.hudi.common.table.view; -import com.uber.hoodie.common.model.CompactionOperation; -import com.uber.hoodie.common.model.FileSlice; -import com.uber.hoodie.common.model.HoodieDataFile; -import com.uber.hoodie.common.model.HoodieFileGroup; -import com.uber.hoodie.common.table.HoodieTimeline; -import com.uber.hoodie.common.table.SyncableFileSystemView; -import com.uber.hoodie.common.table.timeline.HoodieInstant; -import com.uber.hoodie.common.util.Functions.Function0; -import com.uber.hoodie.common.util.Functions.Function1; -import com.uber.hoodie.common.util.Functions.Function2; -import com.uber.hoodie.common.util.Functions.Function3; -import com.uber.hoodie.common.util.Option; -import com.uber.hoodie.common.util.collection.Pair; import java.io.Serializable; import java.util.List; import java.util.stream.Stream; +import org.apache.hudi.common.model.CompactionOperation; +import org.apache.hudi.common.model.FileSlice; +import org.apache.hudi.common.model.HoodieDataFile; +import org.apache.hudi.common.model.HoodieFileGroup; +import org.apache.hudi.common.table.HoodieTimeline; +import org.apache.hudi.common.table.SyncableFileSystemView; +import org.apache.hudi.common.table.timeline.HoodieInstant; +import org.apache.hudi.common.util.Functions.Function0; +import org.apache.hudi.common.util.Functions.Function1; +import org.apache.hudi.common.util.Functions.Function2; +import org.apache.hudi.common.util.Functions.Function3; +import org.apache.hudi.common.util.Option; +import org.apache.hudi.common.util.collection.Pair; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; diff --git a/hoodie-common/src/main/java/com/uber/hoodie/common/table/view/RemoteHoodieTableFileSystemView.java b/hudi-common/src/main/java/org/apache/hudi/common/table/view/RemoteHoodieTableFileSystemView.java similarity index 94% rename from hoodie-common/src/main/java/com/uber/hoodie/common/table/view/RemoteHoodieTableFileSystemView.java rename to hudi-common/src/main/java/org/apache/hudi/common/table/view/RemoteHoodieTableFileSystemView.java index 9d6972f79..7084acde0 100644 --- a/hoodie-common/src/main/java/com/uber/hoodie/common/table/view/RemoteHoodieTableFileSystemView.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/view/RemoteHoodieTableFileSystemView.java @@ -16,29 +16,11 @@ * limitations under the License. */ -package com.uber.hoodie.common.table.view; +package org.apache.hudi.common.table.view; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Preconditions; -import com.uber.hoodie.common.model.CompactionOperation; -import com.uber.hoodie.common.model.FileSlice; -import com.uber.hoodie.common.model.HoodieDataFile; -import com.uber.hoodie.common.model.HoodieFileGroup; -import com.uber.hoodie.common.table.HoodieTableMetaClient; -import com.uber.hoodie.common.table.HoodieTimeline; -import com.uber.hoodie.common.table.SyncableFileSystemView; -import com.uber.hoodie.common.table.timeline.HoodieInstant; -import com.uber.hoodie.common.table.timeline.dto.CompactionOpDTO; -import com.uber.hoodie.common.table.timeline.dto.DataFileDTO; -import com.uber.hoodie.common.table.timeline.dto.FileGroupDTO; -import com.uber.hoodie.common.table.timeline.dto.FileSliceDTO; -import com.uber.hoodie.common.table.timeline.dto.InstantDTO; -import com.uber.hoodie.common.table.timeline.dto.TimelineDTO; -import com.uber.hoodie.common.util.Option; -import com.uber.hoodie.common.util.StringUtils; -import com.uber.hoodie.common.util.collection.Pair; -import com.uber.hoodie.exception.HoodieRemoteException; import java.io.IOException; import java.io.Serializable; import java.util.HashMap; @@ -48,6 +30,24 @@ import java.util.stream.Stream; import org.apache.http.client.fluent.Request; import org.apache.http.client.fluent.Response; import org.apache.http.client.utils.URIBuilder; +import org.apache.hudi.common.model.CompactionOperation; +import org.apache.hudi.common.model.FileSlice; +import org.apache.hudi.common.model.HoodieDataFile; +import org.apache.hudi.common.model.HoodieFileGroup; +import org.apache.hudi.common.table.HoodieTableMetaClient; +import org.apache.hudi.common.table.HoodieTimeline; +import org.apache.hudi.common.table.SyncableFileSystemView; +import org.apache.hudi.common.table.timeline.HoodieInstant; +import org.apache.hudi.common.table.timeline.dto.CompactionOpDTO; +import org.apache.hudi.common.table.timeline.dto.DataFileDTO; +import org.apache.hudi.common.table.timeline.dto.FileGroupDTO; +import org.apache.hudi.common.table.timeline.dto.FileSliceDTO; +import org.apache.hudi.common.table.timeline.dto.InstantDTO; +import org.apache.hudi.common.table.timeline.dto.TimelineDTO; +import org.apache.hudi.common.util.Option; +import org.apache.hudi.common.util.StringUtils; +import org.apache.hudi.common.util.collection.Pair; +import org.apache.hudi.exception.HoodieRemoteException; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; diff --git a/hoodie-common/src/main/java/com/uber/hoodie/common/table/view/RocksDbBasedFileSystemView.java b/hudi-common/src/main/java/org/apache/hudi/common/table/view/RocksDbBasedFileSystemView.java similarity index 95% rename from hoodie-common/src/main/java/com/uber/hoodie/common/table/view/RocksDbBasedFileSystemView.java rename to hudi-common/src/main/java/org/apache/hudi/common/table/view/RocksDbBasedFileSystemView.java index ab598e881..6d2e1c8fa 100644 --- a/hoodie-common/src/main/java/com/uber/hoodie/common/table/view/RocksDbBasedFileSystemView.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/view/RocksDbBasedFileSystemView.java @@ -16,21 +16,9 @@ * limitations under the License. */ -package com.uber.hoodie.common.table.view; +package org.apache.hudi.common.table.view; import com.google.common.base.Preconditions; -import com.uber.hoodie.common.model.CompactionOperation; -import com.uber.hoodie.common.model.FileSlice; -import com.uber.hoodie.common.model.HoodieDataFile; -import com.uber.hoodie.common.model.HoodieFileGroup; -import com.uber.hoodie.common.model.HoodieFileGroupId; -import com.uber.hoodie.common.model.HoodieLogFile; -import com.uber.hoodie.common.table.HoodieTableMetaClient; -import com.uber.hoodie.common.table.HoodieTimeline; -import com.uber.hoodie.common.util.Option; -import com.uber.hoodie.common.util.RocksDBDAO; -import com.uber.hoodie.common.util.RocksDBSchemaHelper; -import com.uber.hoodie.common.util.collection.Pair; import java.io.Serializable; import java.util.HashMap; import java.util.List; @@ -40,6 +28,18 @@ import java.util.stream.Collectors; import java.util.stream.Stream; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.Path; +import org.apache.hudi.common.model.CompactionOperation; +import org.apache.hudi.common.model.FileSlice; +import org.apache.hudi.common.model.HoodieDataFile; +import org.apache.hudi.common.model.HoodieFileGroup; +import org.apache.hudi.common.model.HoodieFileGroupId; +import org.apache.hudi.common.model.HoodieLogFile; +import org.apache.hudi.common.table.HoodieTableMetaClient; +import org.apache.hudi.common.table.HoodieTimeline; +import org.apache.hudi.common.util.Option; +import org.apache.hudi.common.util.RocksDBDAO; +import org.apache.hudi.common.util.RocksDBSchemaHelper; +import org.apache.hudi.common.util.collection.Pair; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; diff --git a/hoodie-common/src/main/java/com/uber/hoodie/common/table/view/SpillableMapBasedFileSystemView.java b/hudi-common/src/main/java/org/apache/hudi/common/table/view/SpillableMapBasedFileSystemView.java similarity index 89% rename from hoodie-common/src/main/java/com/uber/hoodie/common/table/view/SpillableMapBasedFileSystemView.java rename to hudi-common/src/main/java/org/apache/hudi/common/table/view/SpillableMapBasedFileSystemView.java index 0e7fc3408..9ab5696f0 100644 --- a/hoodie-common/src/main/java/com/uber/hoodie/common/table/view/SpillableMapBasedFileSystemView.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/view/SpillableMapBasedFileSystemView.java @@ -16,22 +16,22 @@ * limitations under the License. */ -package com.uber.hoodie.common.table.view; +package org.apache.hudi.common.table.view; -import com.uber.hoodie.common.model.CompactionOperation; -import com.uber.hoodie.common.model.HoodieFileGroup; -import com.uber.hoodie.common.model.HoodieFileGroupId; -import com.uber.hoodie.common.table.HoodieTableMetaClient; -import com.uber.hoodie.common.table.HoodieTimeline; -import com.uber.hoodie.common.util.DefaultSizeEstimator; -import com.uber.hoodie.common.util.collection.ExternalSpillableMap; -import com.uber.hoodie.common.util.collection.Pair; import java.io.File; import java.io.IOException; import java.util.List; import java.util.Map; import java.util.stream.Stream; import org.apache.hadoop.fs.FileStatus; +import org.apache.hudi.common.model.CompactionOperation; +import org.apache.hudi.common.model.HoodieFileGroup; +import org.apache.hudi.common.model.HoodieFileGroupId; +import org.apache.hudi.common.table.HoodieTableMetaClient; +import org.apache.hudi.common.table.HoodieTimeline; +import org.apache.hudi.common.util.DefaultSizeEstimator; +import org.apache.hudi.common.util.collection.ExternalSpillableMap; +import org.apache.hudi.common.util.collection.Pair; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; diff --git a/hoodie-common/src/main/java/com/uber/hoodie/common/util/AvroUtils.java b/hudi-common/src/main/java/org/apache/hudi/common/util/AvroUtils.java similarity index 92% rename from hoodie-common/src/main/java/com/uber/hoodie/common/util/AvroUtils.java rename to hudi-common/src/main/java/org/apache/hudi/common/util/AvroUtils.java index ade9248e9..01baa4719 100644 --- a/hoodie-common/src/main/java/com/uber/hoodie/common/util/AvroUtils.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/util/AvroUtils.java @@ -16,25 +16,11 @@ * limitations under the License. */ -package com.uber.hoodie.common.util; +package org.apache.hudi.common.util; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; -import com.uber.hoodie.avro.model.HoodieCleanMetadata; -import com.uber.hoodie.avro.model.HoodieCleanPartitionMetadata; -import com.uber.hoodie.avro.model.HoodieCompactionPlan; -import com.uber.hoodie.avro.model.HoodieRestoreMetadata; -import com.uber.hoodie.avro.model.HoodieRollbackMetadata; -import com.uber.hoodie.avro.model.HoodieRollbackPartitionMetadata; -import com.uber.hoodie.avro.model.HoodieSavepointMetadata; -import com.uber.hoodie.avro.model.HoodieSavepointPartitionMetadata; -import com.uber.hoodie.common.HoodieCleanStat; -import com.uber.hoodie.common.HoodieRollbackStat; -import com.uber.hoodie.common.model.HoodieAvroPayload; -import com.uber.hoodie.common.model.HoodieKey; -import com.uber.hoodie.common.model.HoodieRecord; -import com.uber.hoodie.exception.HoodieIOException; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.util.Arrays; @@ -56,6 +42,20 @@ import org.apache.avro.specific.SpecificDatumWriter; import org.apache.avro.specific.SpecificRecordBase; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hudi.avro.model.HoodieCleanMetadata; +import org.apache.hudi.avro.model.HoodieCleanPartitionMetadata; +import org.apache.hudi.avro.model.HoodieCompactionPlan; +import org.apache.hudi.avro.model.HoodieRestoreMetadata; +import org.apache.hudi.avro.model.HoodieRollbackMetadata; +import org.apache.hudi.avro.model.HoodieRollbackPartitionMetadata; +import org.apache.hudi.avro.model.HoodieSavepointMetadata; +import org.apache.hudi.avro.model.HoodieSavepointPartitionMetadata; +import org.apache.hudi.common.HoodieCleanStat; +import org.apache.hudi.common.HoodieRollbackStat; +import org.apache.hudi.common.model.HoodieAvroPayload; +import org.apache.hudi.common.model.HoodieKey; +import org.apache.hudi.common.model.HoodieRecord; +import org.apache.hudi.exception.HoodieIOException; public class AvroUtils { diff --git a/hoodie-common/src/main/java/com/uber/hoodie/common/util/CompactionUtils.java b/hudi-common/src/main/java/org/apache/hudi/common/util/CompactionUtils.java similarity index 92% rename from hoodie-common/src/main/java/com/uber/hoodie/common/util/CompactionUtils.java rename to hudi-common/src/main/java/org/apache/hudi/common/util/CompactionUtils.java index 359a0e8ef..dfecf28eb 100644 --- a/hoodie-common/src/main/java/com/uber/hoodie/common/util/CompactionUtils.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/util/CompactionUtils.java @@ -16,18 +16,8 @@ * limitations under the License. */ -package com.uber.hoodie.common.util; +package org.apache.hudi.common.util; -import com.uber.hoodie.avro.model.HoodieCompactionOperation; -import com.uber.hoodie.avro.model.HoodieCompactionPlan; -import com.uber.hoodie.common.model.CompactionOperation; -import com.uber.hoodie.common.model.FileSlice; -import com.uber.hoodie.common.model.HoodieFileGroupId; -import com.uber.hoodie.common.table.HoodieTableMetaClient; -import com.uber.hoodie.common.table.HoodieTimeline; -import com.uber.hoodie.common.table.timeline.HoodieInstant; -import com.uber.hoodie.common.util.collection.Pair; -import com.uber.hoodie.exception.HoodieException; import java.io.IOException; import java.util.HashMap; import java.util.List; @@ -35,6 +25,16 @@ import java.util.Map; import java.util.function.Function; import java.util.stream.Collectors; import java.util.stream.Stream; +import org.apache.hudi.avro.model.HoodieCompactionOperation; +import org.apache.hudi.avro.model.HoodieCompactionPlan; +import org.apache.hudi.common.model.CompactionOperation; +import org.apache.hudi.common.model.FileSlice; +import org.apache.hudi.common.model.HoodieFileGroupId; +import org.apache.hudi.common.table.HoodieTableMetaClient; +import org.apache.hudi.common.table.HoodieTimeline; +import org.apache.hudi.common.table.timeline.HoodieInstant; +import org.apache.hudi.common.util.collection.Pair; +import org.apache.hudi.exception.HoodieException; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; diff --git a/hoodie-common/src/main/java/com/uber/hoodie/common/util/ConsistencyGuard.java b/hudi-common/src/main/java/org/apache/hudi/common/util/ConsistencyGuard.java similarity index 98% rename from hoodie-common/src/main/java/com/uber/hoodie/common/util/ConsistencyGuard.java rename to hudi-common/src/main/java/org/apache/hudi/common/util/ConsistencyGuard.java index 7597c57bf..57be07b5f 100644 --- a/hoodie-common/src/main/java/com/uber/hoodie/common/util/ConsistencyGuard.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/util/ConsistencyGuard.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package com.uber.hoodie.common.util; +package org.apache.hudi.common.util; import java.io.IOException; import java.util.List; diff --git a/hoodie-common/src/main/java/com/uber/hoodie/common/util/ConsistencyGuardConfig.java b/hudi-common/src/main/java/org/apache/hudi/common/util/ConsistencyGuardConfig.java similarity index 98% rename from hoodie-common/src/main/java/com/uber/hoodie/common/util/ConsistencyGuardConfig.java rename to hudi-common/src/main/java/org/apache/hudi/common/util/ConsistencyGuardConfig.java index 14181919d..a73fc5ec9 100644 --- a/hoodie-common/src/main/java/com/uber/hoodie/common/util/ConsistencyGuardConfig.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/util/ConsistencyGuardConfig.java @@ -16,13 +16,13 @@ * limitations under the License. */ -package com.uber.hoodie.common.util; +package org.apache.hudi.common.util; -import com.uber.hoodie.config.DefaultHoodieConfig; import java.io.File; import java.io.FileReader; import java.io.IOException; import java.util.Properties; +import org.apache.hudi.config.DefaultHoodieConfig; public class ConsistencyGuardConfig extends DefaultHoodieConfig { diff --git a/hoodie-common/src/main/java/com/uber/hoodie/common/util/DFSPropertiesConfiguration.java b/hudi-common/src/main/java/org/apache/hudi/common/util/DFSPropertiesConfiguration.java similarity index 99% rename from hoodie-common/src/main/java/com/uber/hoodie/common/util/DFSPropertiesConfiguration.java rename to hudi-common/src/main/java/org/apache/hudi/common/util/DFSPropertiesConfiguration.java index c5f803be1..982098e36 100644 --- a/hoodie-common/src/main/java/com/uber/hoodie/common/util/DFSPropertiesConfiguration.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/util/DFSPropertiesConfiguration.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package com.uber.hoodie.common.util; +package org.apache.hudi.common.util; import java.io.BufferedReader; import java.io.IOException; diff --git a/hoodie-common/src/main/java/com/uber/hoodie/common/util/DefaultSizeEstimator.java b/hudi-common/src/main/java/org/apache/hudi/common/util/DefaultSizeEstimator.java similarity index 96% rename from hoodie-common/src/main/java/com/uber/hoodie/common/util/DefaultSizeEstimator.java rename to hudi-common/src/main/java/org/apache/hudi/common/util/DefaultSizeEstimator.java index b56ff237b..4c8320416 100644 --- a/hoodie-common/src/main/java/com/uber/hoodie/common/util/DefaultSizeEstimator.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/util/DefaultSizeEstimator.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package com.uber.hoodie.common.util; +package org.apache.hudi.common.util; import com.twitter.common.objectsize.ObjectSizeCalculator; diff --git a/hoodie-common/src/main/java/com/uber/hoodie/common/util/FSUtils.java b/hudi-common/src/main/java/org/apache/hudi/common/util/FSUtils.java similarity index 95% rename from hoodie-common/src/main/java/com/uber/hoodie/common/util/FSUtils.java rename to hudi-common/src/main/java/org/apache/hudi/common/util/FSUtils.java index 2df24a08a..df2054777 100644 --- a/hoodie-common/src/main/java/com/uber/hoodie/common/util/FSUtils.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/util/FSUtils.java @@ -16,21 +16,10 @@ * limitations under the License. */ -package com.uber.hoodie.common.util; - -import static com.uber.hoodie.common.table.HoodieTableMetaClient.MARKER_EXTN; +package org.apache.hudi.common.util; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; -import com.uber.hoodie.common.model.HoodieFileFormat; -import com.uber.hoodie.common.model.HoodieLogFile; -import com.uber.hoodie.common.model.HoodiePartitionMetadata; -import com.uber.hoodie.common.table.HoodieTableMetaClient; -import com.uber.hoodie.common.table.timeline.HoodieInstant; -import com.uber.hoodie.common.util.collection.Pair; -import com.uber.hoodie.exception.HoodieException; -import com.uber.hoodie.exception.HoodieIOException; -import com.uber.hoodie.exception.InvalidHoodiePathException; import java.io.File; import java.io.IOException; import java.util.ArrayList; @@ -52,6 +41,15 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.PathFilter; import org.apache.hadoop.fs.RemoteIterator; import org.apache.hadoop.hdfs.DistributedFileSystem; +import org.apache.hudi.common.model.HoodieFileFormat; +import org.apache.hudi.common.model.HoodieLogFile; +import org.apache.hudi.common.model.HoodiePartitionMetadata; +import org.apache.hudi.common.table.HoodieTableMetaClient; +import org.apache.hudi.common.table.timeline.HoodieInstant; +import org.apache.hudi.common.util.collection.Pair; +import org.apache.hudi.exception.HoodieException; +import org.apache.hudi.exception.HoodieIOException; +import org.apache.hudi.exception.InvalidHoodiePathException; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; @@ -122,18 +120,18 @@ public class FSUtils { } public static String makeMarkerFile(String commitTime, String writeToken, String fileId) { - return String.format("%s_%s_%s%s", fileId, writeToken, commitTime, MARKER_EXTN); + return String.format("%s_%s_%s%s", fileId, writeToken, commitTime, HoodieTableMetaClient.MARKER_EXTN); } public static String translateMarkerToDataPath(String basePath, String markerPath, String instantTs) { - Preconditions.checkArgument(markerPath.endsWith(MARKER_EXTN)); + Preconditions.checkArgument(markerPath.endsWith(HoodieTableMetaClient.MARKER_EXTN)); String markerRootPath = Path.getPathWithoutSchemeAndAuthority(new Path( String.format("%s/%s/%s", basePath, HoodieTableMetaClient.TEMPFOLDER_NAME, instantTs))).toString(); int begin = markerPath.indexOf(markerRootPath); Preconditions.checkArgument(begin >= 0, "Not in marker dir. Marker Path=" + markerPath + ", Expected Marker Root=" + markerRootPath); String rPath = markerPath.substring(begin + markerRootPath.length() + 1); - return String.format("%s/%s%s", basePath, rPath.replace(MARKER_EXTN, ""), + return String.format("%s/%s%s", basePath, rPath.replace(HoodieTableMetaClient.MARKER_EXTN, ""), HoodieFileFormat.PARQUET.getFileExtension()); } @@ -193,7 +191,7 @@ public class FSUtils { /** * Obtain all the partition paths, that are present in this table, denoted by presence of {@link - * com.uber.hoodie.common.model.HoodiePartitionMetadata#HOODIE_PARTITION_METAFILE} + * HoodiePartitionMetadata#HOODIE_PARTITION_METAFILE} */ public static List getAllFoldersWithPartitionMetaFile(FileSystem fs, String basePathStr) throws IOException { @@ -214,7 +212,7 @@ public class FSUtils { List dataFiles = new LinkedList<>(); processFiles(fs, markerDir, (status) -> { String pathStr = status.getPath().toString(); - if (pathStr.endsWith(MARKER_EXTN)) { + if (pathStr.endsWith(HoodieTableMetaClient.MARKER_EXTN)) { dataFiles.add(FSUtils.translateMarkerToDataPath(basePath, pathStr, instantTs)); } return true; diff --git a/hoodie-common/src/main/java/com/uber/hoodie/common/util/FailSafeConsistencyGuard.java b/hudi-common/src/main/java/org/apache/hudi/common/util/FailSafeConsistencyGuard.java similarity index 99% rename from hoodie-common/src/main/java/com/uber/hoodie/common/util/FailSafeConsistencyGuard.java rename to hudi-common/src/main/java/org/apache/hudi/common/util/FailSafeConsistencyGuard.java index ee215be19..e1b816ec0 100644 --- a/hoodie-common/src/main/java/com/uber/hoodie/common/util/FailSafeConsistencyGuard.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/util/FailSafeConsistencyGuard.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package com.uber.hoodie.common.util; +package org.apache.hudi.common.util; import com.google.common.base.Preconditions; import java.io.FileNotFoundException; diff --git a/hoodie-common/src/main/java/com/uber/hoodie/common/util/Functions.java b/hudi-common/src/main/java/org/apache/hudi/common/util/Functions.java similarity index 97% rename from hoodie-common/src/main/java/com/uber/hoodie/common/util/Functions.java rename to hudi-common/src/main/java/org/apache/hudi/common/util/Functions.java index 13b95ece8..035124022 100644 --- a/hoodie-common/src/main/java/com/uber/hoodie/common/util/Functions.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/util/Functions.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package com.uber.hoodie.common.util; +package org.apache.hudi.common.util; import java.io.Serializable; diff --git a/hoodie-common/src/main/java/com/uber/hoodie/common/util/HoodieAvroUtils.java b/hudi-common/src/main/java/org/apache/hudi/common/util/HoodieAvroUtils.java similarity index 97% rename from hoodie-common/src/main/java/com/uber/hoodie/common/util/HoodieAvroUtils.java rename to hudi-common/src/main/java/org/apache/hudi/common/util/HoodieAvroUtils.java index 3a9443e37..75b9b556a 100644 --- a/hoodie-common/src/main/java/com/uber/hoodie/common/util/HoodieAvroUtils.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/util/HoodieAvroUtils.java @@ -16,11 +16,8 @@ * limitations under the License. */ -package com.uber.hoodie.common.util; +package org.apache.hudi.common.util; -import com.uber.hoodie.common.model.HoodieRecord; -import com.uber.hoodie.exception.HoodieIOException; -import com.uber.hoodie.exception.SchemaCompatabilityException; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; @@ -43,6 +40,9 @@ import org.apache.avro.io.BinaryDecoder; import org.apache.avro.io.BinaryEncoder; import org.apache.avro.io.DecoderFactory; import org.apache.avro.io.EncoderFactory; +import org.apache.hudi.common.model.HoodieRecord; +import org.apache.hudi.exception.HoodieIOException; +import org.apache.hudi.exception.SchemaCompatabilityException; import org.codehaus.jackson.JsonNode; import org.codehaus.jackson.node.NullNode; diff --git a/hoodie-common/src/main/java/com/uber/hoodie/common/util/HoodieRecordSizeEstimator.java b/hudi-common/src/main/java/org/apache/hudi/common/util/HoodieRecordSizeEstimator.java similarity index 89% rename from hoodie-common/src/main/java/com/uber/hoodie/common/util/HoodieRecordSizeEstimator.java rename to hudi-common/src/main/java/org/apache/hudi/common/util/HoodieRecordSizeEstimator.java index ed024ed7a..3a29b9aa6 100644 --- a/hoodie-common/src/main/java/com/uber/hoodie/common/util/HoodieRecordSizeEstimator.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/util/HoodieRecordSizeEstimator.java @@ -16,12 +16,13 @@ * limitations under the License. */ -package com.uber.hoodie.common.util; +package org.apache.hudi.common.util; import com.twitter.common.objectsize.ObjectSizeCalculator; -import com.uber.hoodie.common.model.HoodieRecord; -import com.uber.hoodie.common.model.HoodieRecordPayload; import org.apache.avro.Schema; +import org.apache.hudi.common.model.HoodieRecord; +import org.apache.hudi.common.model.HoodieRecordPayload; +import org.apache.hudi.common.util.collection.ExternalSpillableMap; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; @@ -45,7 +46,7 @@ public class HoodieRecordSizeEstimator implements // Most HoodieRecords are bound to have data + schema. Although, the same schema object is shared amongst // all records in the JVM. Calculate and print the size of the Schema and of the Record to // note the sizes and differences. A correct estimation in such cases is handled in - /** {@link com.uber.hoodie.common.util.collection.ExternalSpillableMap} **/ + /** {@link ExternalSpillableMap} **/ long sizeOfRecord = ObjectSizeCalculator.getObjectSize(hoodieRecord); long sizeOfSchema = ObjectSizeCalculator.getObjectSize(schema); log.info("SizeOfRecord => " + sizeOfRecord + " SizeOfSchema => " + sizeOfSchema); diff --git a/hoodie-common/src/main/java/com/uber/hoodie/common/util/HoodieTimer.java b/hudi-common/src/main/java/org/apache/hudi/common/util/HoodieTimer.java similarity index 96% rename from hoodie-common/src/main/java/com/uber/hoodie/common/util/HoodieTimer.java rename to hudi-common/src/main/java/org/apache/hudi/common/util/HoodieTimer.java index c3acbda90..ba1f4e088 100644 --- a/hoodie-common/src/main/java/com/uber/hoodie/common/util/HoodieTimer.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/util/HoodieTimer.java @@ -16,11 +16,11 @@ * limitations under the License. */ -package com.uber.hoodie.common.util; +package org.apache.hudi.common.util; -import com.uber.hoodie.exception.HoodieException; import java.util.ArrayDeque; import java.util.Deque; +import org.apache.hudi.exception.HoodieException; /** * Timing utility to help keep track of execution times of code blocks. This class helps to allow multiple diff --git a/hoodie-common/src/main/java/com/uber/hoodie/common/util/LogReaderUtils.java b/hudi-common/src/main/java/org/apache/hudi/common/util/LogReaderUtils.java similarity index 80% rename from hoodie-common/src/main/java/com/uber/hoodie/common/util/LogReaderUtils.java rename to hudi-common/src/main/java/org/apache/hudi/common/util/LogReaderUtils.java index ad5d500c8..bb21cec57 100644 --- a/hoodie-common/src/main/java/com/uber/hoodie/common/util/LogReaderUtils.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/util/LogReaderUtils.java @@ -16,16 +16,8 @@ * limitations under the License. */ -package com.uber.hoodie.common.util; +package org.apache.hudi.common.util; -import com.uber.hoodie.common.model.HoodieLogFile; -import com.uber.hoodie.common.table.HoodieTableMetaClient; -import com.uber.hoodie.common.table.HoodieTimeline; -import com.uber.hoodie.common.table.log.HoodieLogFormat; -import com.uber.hoodie.common.table.log.block.HoodieAvroDataBlock; -import com.uber.hoodie.common.table.log.block.HoodieLogBlock; -import com.uber.hoodie.common.table.log.block.HoodieLogBlock.HeaderMetadataType; -import com.uber.hoodie.common.table.timeline.HoodieActiveTimeline; import java.io.IOException; import java.util.List; import java.util.stream.Collectors; @@ -33,6 +25,15 @@ import org.apache.avro.Schema; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.mapred.JobConf; +import org.apache.hudi.common.model.HoodieLogFile; +import org.apache.hudi.common.table.HoodieTableMetaClient; +import org.apache.hudi.common.table.HoodieTimeline; +import org.apache.hudi.common.table.log.HoodieLogFormat; +import org.apache.hudi.common.table.log.HoodieLogFormat.Reader; +import org.apache.hudi.common.table.log.block.HoodieAvroDataBlock; +import org.apache.hudi.common.table.log.block.HoodieLogBlock; +import org.apache.hudi.common.table.log.block.HoodieLogBlock.HeaderMetadataType; +import org.apache.hudi.common.table.timeline.HoodieActiveTimeline; /** * Utils class for performing various log file reading operations @@ -41,7 +42,7 @@ public class LogReaderUtils { private static Schema readSchemaFromLogFileInReverse(FileSystem fs, HoodieActiveTimeline activeTimeline, Path path) throws IOException { - HoodieLogFormat.Reader reader = HoodieLogFormat.newReader(fs, new HoodieLogFile(path), null, true, true); + Reader reader = HoodieLogFormat.newReader(fs, new HoodieLogFile(path), null, true, true); Schema writerSchema = null; HoodieTimeline completedTimeline = activeTimeline.getCommitsTimeline().filterCompletedInstants(); while (reader.hasPrev()) { diff --git a/hoodie-common/src/main/java/com/uber/hoodie/common/util/NetworkUtils.java b/hudi-common/src/main/java/org/apache/hudi/common/util/NetworkUtils.java similarity index 94% rename from hoodie-common/src/main/java/com/uber/hoodie/common/util/NetworkUtils.java rename to hudi-common/src/main/java/org/apache/hudi/common/util/NetworkUtils.java index 0cf476020..1e82d719f 100644 --- a/hoodie-common/src/main/java/com/uber/hoodie/common/util/NetworkUtils.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/util/NetworkUtils.java @@ -16,11 +16,11 @@ * limitations under the License. */ -package com.uber.hoodie.common.util; +package org.apache.hudi.common.util; -import com.uber.hoodie.exception.HoodieException; import java.io.IOException; import java.net.ServerSocket; +import org.apache.hudi.exception.HoodieException; public class NetworkUtils { diff --git a/hoodie-common/src/main/java/com/uber/hoodie/common/util/NoOpConsistencyGuard.java b/hudi-common/src/main/java/org/apache/hudi/common/util/NoOpConsistencyGuard.java similarity index 97% rename from hoodie-common/src/main/java/com/uber/hoodie/common/util/NoOpConsistencyGuard.java rename to hudi-common/src/main/java/org/apache/hudi/common/util/NoOpConsistencyGuard.java index 5b5ab2d16..f7c597658 100644 --- a/hoodie-common/src/main/java/com/uber/hoodie/common/util/NoOpConsistencyGuard.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/util/NoOpConsistencyGuard.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package com.uber.hoodie.common.util; +package org.apache.hudi.common.util; import java.util.List; import org.apache.hadoop.fs.Path; diff --git a/hoodie-common/src/main/java/com/uber/hoodie/common/util/NumericUtils.java b/hudi-common/src/main/java/org/apache/hudi/common/util/NumericUtils.java similarity index 96% rename from hoodie-common/src/main/java/com/uber/hoodie/common/util/NumericUtils.java rename to hudi-common/src/main/java/org/apache/hudi/common/util/NumericUtils.java index 75ca18fff..ef766851f 100644 --- a/hoodie-common/src/main/java/com/uber/hoodie/common/util/NumericUtils.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/util/NumericUtils.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package com.uber.hoodie.common.util; +package org.apache.hudi.common.util; public class NumericUtils { diff --git a/hoodie-common/src/main/java/com/uber/hoodie/common/util/Option.java b/hudi-common/src/main/java/org/apache/hudi/common/util/Option.java similarity index 99% rename from hoodie-common/src/main/java/com/uber/hoodie/common/util/Option.java rename to hudi-common/src/main/java/org/apache/hudi/common/util/Option.java index 06cea9cc1..e0fa6a429 100644 --- a/hoodie-common/src/main/java/com/uber/hoodie/common/util/Option.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/util/Option.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package com.uber.hoodie.common.util; +package org.apache.hudi.common.util; import java.io.Serializable; import java.util.NoSuchElementException; diff --git a/hoodie-common/src/main/java/com/uber/hoodie/common/util/ParquetUtils.java b/hudi-common/src/main/java/org/apache/hudi/common/util/ParquetUtils.java similarity index 93% rename from hoodie-common/src/main/java/com/uber/hoodie/common/util/ParquetUtils.java rename to hudi-common/src/main/java/org/apache/hudi/common/util/ParquetUtils.java index abba2d6c5..ecb80b63d 100644 --- a/hoodie-common/src/main/java/com/uber/hoodie/common/util/ParquetUtils.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/util/ParquetUtils.java @@ -16,16 +16,8 @@ * limitations under the License. */ -package com.uber.hoodie.common.util; +package org.apache.hudi.common.util; -import static com.uber.hoodie.common.util.FSUtils.getFs; - -import com.uber.hoodie.avro.HoodieAvroWriteSupport; -import com.uber.hoodie.common.BloomFilter; -import com.uber.hoodie.common.model.HoodieRecord; -import com.uber.hoodie.exception.HoodieException; -import com.uber.hoodie.exception.HoodieIOException; -import com.uber.hoodie.exception.MetadataNotFoundException; import java.io.IOException; import java.util.ArrayList; import java.util.HashSet; @@ -38,6 +30,12 @@ import org.apache.avro.generic.GenericRecord; import org.apache.commons.collections.CollectionUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; +import org.apache.hudi.avro.HoodieAvroWriteSupport; +import org.apache.hudi.common.BloomFilter; +import org.apache.hudi.common.model.HoodieRecord; +import org.apache.hudi.exception.HoodieException; +import org.apache.hudi.exception.HoodieIOException; +import org.apache.hudi.exception.MetadataNotFoundException; import org.apache.parquet.avro.AvroParquetReader; import org.apache.parquet.avro.AvroReadSupport; import org.apache.parquet.avro.AvroSchemaConverter; @@ -77,7 +75,7 @@ public class ParquetUtils { filterFunction = Option.of(new RecordKeysFilterFunction(filter)); } Configuration conf = new Configuration(configuration); - conf.addResource(getFs(filePath.toString(), conf).getConf()); + conf.addResource(FSUtils.getFs(filePath.toString(), conf).getConf()); Schema readSchema = HoodieAvroUtils.getRecordKeySchema(); AvroReadSupport.setAvroReadSchema(conf, readSchema); AvroReadSupport.setRequestedProjection(conf, readSchema); @@ -106,7 +104,7 @@ public class ParquetUtils { try { // TODO(vc): Should we use the parallel reading version here? footer = ParquetFileReader - .readFooter(getFs(parquetFilePath.toString(), conf).getConf(), parquetFilePath); + .readFooter(FSUtils.getFs(parquetFilePath.toString(), conf).getConf(), parquetFilePath); } catch (IOException e) { throw new HoodieIOException("Failed to read footer for parquet " + parquetFilePath, e); diff --git a/hoodie-common/src/main/java/com/uber/hoodie/common/util/ReflectionUtils.java b/hudi-common/src/main/java/org/apache/hudi/common/util/ReflectionUtils.java similarity index 96% rename from hoodie-common/src/main/java/com/uber/hoodie/common/util/ReflectionUtils.java rename to hudi-common/src/main/java/org/apache/hudi/common/util/ReflectionUtils.java index a8cfa3eb8..54b51530b 100644 --- a/hoodie-common/src/main/java/com/uber/hoodie/common/util/ReflectionUtils.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/util/ReflectionUtils.java @@ -16,18 +16,18 @@ * limitations under the License. */ -package com.uber.hoodie.common.util; +package org.apache.hudi.common.util; import com.google.common.reflect.ClassPath; import com.google.common.reflect.ClassPath.ClassInfo; -import com.uber.hoodie.common.model.HoodieRecordPayload; -import com.uber.hoodie.exception.HoodieException; import java.io.IOException; import java.lang.reflect.InvocationTargetException; import java.util.Arrays; import java.util.HashMap; import java.util.Map; import java.util.stream.Stream; +import org.apache.hudi.common.model.HoodieRecordPayload; +import org.apache.hudi.exception.HoodieException; public class ReflectionUtils { diff --git a/hoodie-common/src/main/java/com/uber/hoodie/common/util/RocksDBDAO.java b/hudi-common/src/main/java/org/apache/hudi/common/util/RocksDBDAO.java similarity index 98% rename from hoodie-common/src/main/java/com/uber/hoodie/common/util/RocksDBDAO.java rename to hudi-common/src/main/java/org/apache/hudi/common/util/RocksDBDAO.java index 739c778ce..52992a2c6 100644 --- a/hoodie-common/src/main/java/com/uber/hoodie/common/util/RocksDBDAO.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/util/RocksDBDAO.java @@ -16,13 +16,10 @@ * limitations under the License. */ -package com.uber.hoodie.common.util; +package org.apache.hudi.common.util; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; -import com.uber.hoodie.common.util.collection.Pair; -import com.uber.hoodie.exception.HoodieException; -import com.uber.hoodie.exception.HoodieIOException; import java.io.File; import java.io.IOException; import java.io.Serializable; @@ -34,6 +31,9 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.stream.Collectors; import java.util.stream.Stream; import org.apache.commons.io.FileUtils; +import org.apache.hudi.common.util.collection.Pair; +import org.apache.hudi.exception.HoodieException; +import org.apache.hudi.exception.HoodieIOException; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; import org.rocksdb.ColumnFamilyDescriptor; diff --git a/hoodie-common/src/main/java/com/uber/hoodie/common/util/RocksDBSchemaHelper.java b/hudi-common/src/main/java/org/apache/hudi/common/util/RocksDBSchemaHelper.java similarity index 94% rename from hoodie-common/src/main/java/com/uber/hoodie/common/util/RocksDBSchemaHelper.java rename to hudi-common/src/main/java/org/apache/hudi/common/util/RocksDBSchemaHelper.java index e7c0b2c7b..c14df29a2 100644 --- a/hoodie-common/src/main/java/com/uber/hoodie/common/util/RocksDBSchemaHelper.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/util/RocksDBSchemaHelper.java @@ -16,14 +16,14 @@ * limitations under the License. */ -package com.uber.hoodie.common.util; +package org.apache.hudi.common.util; -import com.uber.hoodie.common.model.FileSlice; -import com.uber.hoodie.common.model.HoodieFileGroup; -import com.uber.hoodie.common.model.HoodieFileGroupId; -import com.uber.hoodie.common.table.HoodieTableMetaClient; import java.util.Arrays; import java.util.List; +import org.apache.hudi.common.model.FileSlice; +import org.apache.hudi.common.model.HoodieFileGroup; +import org.apache.hudi.common.model.HoodieFileGroupId; +import org.apache.hudi.common.table.HoodieTableMetaClient; /** * Helper class to generate Key and column names for rocksdb based view diff --git a/hoodie-common/src/main/java/com/uber/hoodie/common/util/SerializationUtils.java b/hudi-common/src/main/java/org/apache/hudi/common/util/SerializationUtils.java similarity index 98% rename from hoodie-common/src/main/java/com/uber/hoodie/common/util/SerializationUtils.java rename to hudi-common/src/main/java/org/apache/hudi/common/util/SerializationUtils.java index ae5b645a2..d5bc6c56a 100644 --- a/hoodie-common/src/main/java/com/uber/hoodie/common/util/SerializationUtils.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/util/SerializationUtils.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package com.uber.hoodie.common.util; +package org.apache.hudi.common.util; import com.esotericsoftware.kryo.Kryo; import com.esotericsoftware.kryo.Serializer; @@ -24,12 +24,12 @@ import com.esotericsoftware.kryo.io.Input; import com.esotericsoftware.kryo.io.Output; import com.esotericsoftware.kryo.serializers.FieldSerializer; import com.esotericsoftware.reflectasm.ConstructorAccess; -import com.uber.hoodie.exception.HoodieSerializationException; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.Serializable; import java.lang.reflect.Constructor; import java.lang.reflect.InvocationTargetException; +import org.apache.hudi.exception.HoodieSerializationException; import org.objenesis.instantiator.ObjectInstantiator; diff --git a/hoodie-common/src/main/java/com/uber/hoodie/common/util/SizeEstimator.java b/hudi-common/src/main/java/org/apache/hudi/common/util/SizeEstimator.java similarity index 96% rename from hoodie-common/src/main/java/com/uber/hoodie/common/util/SizeEstimator.java rename to hudi-common/src/main/java/org/apache/hudi/common/util/SizeEstimator.java index c21c4aec0..63fc9aaa3 100644 --- a/hoodie-common/src/main/java/com/uber/hoodie/common/util/SizeEstimator.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/util/SizeEstimator.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package com.uber.hoodie.common.util; +package org.apache.hudi.common.util; /** * An interface to estimate the size of payload in memory diff --git a/hoodie-common/src/main/java/com/uber/hoodie/common/util/SpillableMapUtils.java b/hudi-common/src/main/java/org/apache/hudi/common/util/SpillableMapUtils.java similarity index 85% rename from hoodie-common/src/main/java/com/uber/hoodie/common/util/SpillableMapUtils.java rename to hudi-common/src/main/java/org/apache/hudi/common/util/SpillableMapUtils.java index 13d80585a..f9d9cb7f0 100644 --- a/hoodie-common/src/main/java/com/uber/hoodie/common/util/SpillableMapUtils.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/util/SpillableMapUtils.java @@ -16,18 +16,18 @@ * limitations under the License. */ -package com.uber.hoodie.common.util; +package org.apache.hudi.common.util; -import com.uber.hoodie.common.model.HoodieKey; -import com.uber.hoodie.common.model.HoodieRecord; -import com.uber.hoodie.common.model.HoodieRecordPayload; -import com.uber.hoodie.common.util.collection.DiskBasedMap; -import com.uber.hoodie.common.util.collection.io.storage.SizeAwareDataOutputStream; -import com.uber.hoodie.exception.HoodieCorruptedDataException; import java.io.IOException; import java.io.RandomAccessFile; import java.util.zip.CRC32; import org.apache.avro.generic.GenericRecord; +import org.apache.hudi.common.model.HoodieKey; +import org.apache.hudi.common.model.HoodieRecord; +import org.apache.hudi.common.model.HoodieRecordPayload; +import org.apache.hudi.common.util.collection.DiskBasedMap.FileEntry; +import org.apache.hudi.common.util.collection.io.storage.SizeAwareDataOutputStream; +import org.apache.hudi.exception.HoodieCorruptedDataException; public class SpillableMapUtils { @@ -36,14 +36,14 @@ public class SpillableMapUtils { */ public static byte[] readBytesFromDisk(RandomAccessFile file, long valuePosition, int valueLength) throws IOException { - DiskBasedMap.FileEntry fileEntry = readInternal(file, valuePosition, valueLength); + FileEntry fileEntry = readInternal(file, valuePosition, valueLength); return fileEntry.getValue(); } /** * |crc|timestamp|sizeOfKey|SizeOfValue|key|value| */ - private static DiskBasedMap.FileEntry readInternal(RandomAccessFile file, long valuePosition, + private static FileEntry readInternal(RandomAccessFile file, long valuePosition, int valueLength) throws IOException { file.seek(valuePosition); long crc = file.readLong(); @@ -62,7 +62,7 @@ public class SpillableMapUtils { throw new HoodieCorruptedDataException("checksum of payload written to external disk does not match, " + "data may be corrupted"); } - return new DiskBasedMap.FileEntry(crc, keySize, valueSize, key, value, timestamp); + return new FileEntry(crc, keySize, valueSize, key, value, timestamp); } /** @@ -70,12 +70,12 @@ public class SpillableMapUtils { * |crc|timestamp|sizeOfKey|SizeOfValue|key|value| */ public static long spillToDisk(SizeAwareDataOutputStream outputStream, - DiskBasedMap.FileEntry fileEntry) throws IOException { + FileEntry fileEntry) throws IOException { return spill(outputStream, fileEntry); } private static long spill(SizeAwareDataOutputStream outputStream, - DiskBasedMap.FileEntry fileEntry) + FileEntry fileEntry) throws IOException { outputStream.writeLong(fileEntry.getCrc()); outputStream.writeLong(fileEntry.getTimestamp()); diff --git a/hoodie-common/src/main/java/com/uber/hoodie/common/util/StringUtils.java b/hudi-common/src/main/java/org/apache/hudi/common/util/StringUtils.java similarity index 98% rename from hoodie-common/src/main/java/com/uber/hoodie/common/util/StringUtils.java rename to hudi-common/src/main/java/org/apache/hudi/common/util/StringUtils.java index 5ac1d0491..d3d162d14 100644 --- a/hoodie-common/src/main/java/com/uber/hoodie/common/util/StringUtils.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/util/StringUtils.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package com.uber.hoodie.common.util; +package org.apache.hudi.common.util; /** * Simple utility for operations on strings diff --git a/hoodie-common/src/main/java/com/uber/hoodie/common/util/TimelineDiffHelper.java b/hudi-common/src/main/java/org/apache/hudi/common/util/TimelineDiffHelper.java similarity index 95% rename from hoodie-common/src/main/java/com/uber/hoodie/common/util/TimelineDiffHelper.java rename to hudi-common/src/main/java/org/apache/hudi/common/util/TimelineDiffHelper.java index 7de28746d..a28162550 100644 --- a/hoodie-common/src/main/java/com/uber/hoodie/common/util/TimelineDiffHelper.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/util/TimelineDiffHelper.java @@ -16,16 +16,16 @@ * limitations under the License. */ -package com.uber.hoodie.common.util; +package org.apache.hudi.common.util; -import com.uber.hoodie.common.table.HoodieTimeline; -import com.uber.hoodie.common.table.timeline.HoodieInstant; -import com.uber.hoodie.common.table.timeline.HoodieInstant.State; -import com.uber.hoodie.common.util.collection.Pair; import java.util.ArrayList; import java.util.List; import java.util.Set; import java.util.stream.Collectors; +import org.apache.hudi.common.table.HoodieTimeline; +import org.apache.hudi.common.table.timeline.HoodieInstant; +import org.apache.hudi.common.table.timeline.HoodieInstant.State; +import org.apache.hudi.common.util.collection.Pair; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; diff --git a/hoodie-common/src/main/java/com/uber/hoodie/common/util/TypedProperties.java b/hudi-common/src/main/java/org/apache/hudi/common/util/TypedProperties.java similarity index 98% rename from hoodie-common/src/main/java/com/uber/hoodie/common/util/TypedProperties.java rename to hudi-common/src/main/java/org/apache/hudi/common/util/TypedProperties.java index b4cc8615d..4f632495d 100644 --- a/hoodie-common/src/main/java/com/uber/hoodie/common/util/TypedProperties.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/util/TypedProperties.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package com.uber.hoodie.common.util; +package org.apache.hudi.common.util; import java.io.Serializable; import java.util.Arrays; diff --git a/hoodie-common/src/main/java/com/uber/hoodie/common/util/collection/DiskBasedMap.java b/hudi-common/src/main/java/org/apache/hudi/common/util/collection/DiskBasedMap.java similarity index 96% rename from hoodie-common/src/main/java/com/uber/hoodie/common/util/collection/DiskBasedMap.java rename to hudi-common/src/main/java/org/apache/hudi/common/util/collection/DiskBasedMap.java index 1b3d7cd7a..57496752a 100644 --- a/hoodie-common/src/main/java/com/uber/hoodie/common/util/collection/DiskBasedMap.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/util/collection/DiskBasedMap.java @@ -16,14 +16,8 @@ * limitations under the License. */ -package com.uber.hoodie.common.util.collection; +package org.apache.hudi.common.util.collection; -import com.uber.hoodie.common.util.SerializationUtils; -import com.uber.hoodie.common.util.SpillableMapUtils; -import com.uber.hoodie.common.util.collection.io.storage.SizeAwareDataOutputStream; -import com.uber.hoodie.exception.HoodieException; -import com.uber.hoodie.exception.HoodieIOException; -import com.uber.hoodie.exception.HoodieNotSupportedException; import java.io.File; import java.io.FileOutputStream; import java.io.IOException; @@ -43,6 +37,12 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.atomic.AtomicLong; import java.util.stream.Stream; +import org.apache.hudi.common.util.SerializationUtils; +import org.apache.hudi.common.util.SpillableMapUtils; +import org.apache.hudi.common.util.collection.io.storage.SizeAwareDataOutputStream; +import org.apache.hudi.exception.HoodieException; +import org.apache.hudi.exception.HoodieIOException; +import org.apache.hudi.exception.HoodieNotSupportedException; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; diff --git a/hoodie-common/src/main/java/com/uber/hoodie/common/util/collection/ExternalSpillableMap.java b/hudi-common/src/main/java/org/apache/hudi/common/util/collection/ExternalSpillableMap.java similarity index 98% rename from hoodie-common/src/main/java/com/uber/hoodie/common/util/collection/ExternalSpillableMap.java rename to hudi-common/src/main/java/org/apache/hudi/common/util/collection/ExternalSpillableMap.java index 91095e245..2b8463780 100644 --- a/hoodie-common/src/main/java/com/uber/hoodie/common/util/collection/ExternalSpillableMap.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/util/collection/ExternalSpillableMap.java @@ -16,11 +16,9 @@ * limitations under the License. */ -package com.uber.hoodie.common.util.collection; +package org.apache.hudi.common.util.collection; import com.twitter.common.objectsize.ObjectSizeCalculator; -import com.uber.hoodie.common.util.SizeEstimator; -import com.uber.hoodie.exception.HoodieIOException; import java.io.IOException; import java.io.Serializable; import java.util.ArrayList; @@ -32,6 +30,8 @@ import java.util.List; import java.util.Map; import java.util.Set; import java.util.stream.Stream; +import org.apache.hudi.common.util.SizeEstimator; +import org.apache.hudi.exception.HoodieIOException; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; diff --git a/hoodie-common/src/main/java/com/uber/hoodie/common/util/collection/ImmutablePair.java b/hudi-common/src/main/java/org/apache/hudi/common/util/collection/ImmutablePair.java similarity index 98% rename from hoodie-common/src/main/java/com/uber/hoodie/common/util/collection/ImmutablePair.java rename to hudi-common/src/main/java/org/apache/hudi/common/util/collection/ImmutablePair.java index d868cd55a..5d340d130 100644 --- a/hoodie-common/src/main/java/com/uber/hoodie/common/util/collection/ImmutablePair.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/util/collection/ImmutablePair.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package com.uber.hoodie.common.util.collection; +package org.apache.hudi.common.util.collection; /** * (NOTE: Adapted from Apache commons-lang3) diff --git a/hoodie-common/src/main/java/com/uber/hoodie/common/util/collection/ImmutableTriple.java b/hudi-common/src/main/java/org/apache/hudi/common/util/collection/ImmutableTriple.java similarity index 98% rename from hoodie-common/src/main/java/com/uber/hoodie/common/util/collection/ImmutableTriple.java rename to hudi-common/src/main/java/org/apache/hudi/common/util/collection/ImmutableTriple.java index 51ea6639f..fc94ed241 100644 --- a/hoodie-common/src/main/java/com/uber/hoodie/common/util/collection/ImmutableTriple.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/util/collection/ImmutableTriple.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package com.uber.hoodie.common.util.collection; +package org.apache.hudi.common.util.collection; /** * (NOTE: Adapted from Apache commons-lang3) diff --git a/hoodie-common/src/main/java/com/uber/hoodie/common/util/collection/LazyFileIterable.java b/hudi-common/src/main/java/org/apache/hudi/common/util/collection/LazyFileIterable.java similarity index 97% rename from hoodie-common/src/main/java/com/uber/hoodie/common/util/collection/LazyFileIterable.java rename to hudi-common/src/main/java/org/apache/hudi/common/util/collection/LazyFileIterable.java index 829ff8258..7ac39c16e 100644 --- a/hoodie-common/src/main/java/com/uber/hoodie/common/util/collection/LazyFileIterable.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/util/collection/LazyFileIterable.java @@ -16,15 +16,15 @@ * limitations under the License. */ -package com.uber.hoodie.common.util.collection; +package org.apache.hudi.common.util.collection; -import com.uber.hoodie.exception.HoodieException; import java.io.IOException; import java.io.RandomAccessFile; import java.util.Iterator; import java.util.Map; import java.util.function.Consumer; import java.util.stream.Collectors; +import org.apache.hudi.exception.HoodieException; /** * Iterable to lazily fetch values spilled to disk. This class uses RandomAccessFile to randomly access the position of diff --git a/hoodie-common/src/main/java/com/uber/hoodie/common/util/collection/Pair.java b/hudi-common/src/main/java/org/apache/hudi/common/util/collection/Pair.java similarity index 99% rename from hoodie-common/src/main/java/com/uber/hoodie/common/util/collection/Pair.java rename to hudi-common/src/main/java/org/apache/hudi/common/util/collection/Pair.java index 4365cfe8e..28fcc982d 100644 --- a/hoodie-common/src/main/java/com/uber/hoodie/common/util/collection/Pair.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/util/collection/Pair.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package com.uber.hoodie.common.util.collection; +package org.apache.hudi.common.util.collection; import java.io.Serializable; import java.util.Map; diff --git a/hoodie-common/src/main/java/com/uber/hoodie/common/util/collection/RocksDBBasedMap.java b/hudi-common/src/main/java/org/apache/hudi/common/util/collection/RocksDBBasedMap.java similarity index 95% rename from hoodie-common/src/main/java/com/uber/hoodie/common/util/collection/RocksDBBasedMap.java rename to hudi-common/src/main/java/org/apache/hudi/common/util/collection/RocksDBBasedMap.java index 267723f37..20d68788c 100644 --- a/hoodie-common/src/main/java/com/uber/hoodie/common/util/collection/RocksDBBasedMap.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/util/collection/RocksDBBasedMap.java @@ -16,15 +16,15 @@ * limitations under the License. */ -package com.uber.hoodie.common.util.collection; +package org.apache.hudi.common.util.collection; -import com.uber.hoodie.common.util.RocksDBDAO; -import com.uber.hoodie.exception.HoodieNotSupportedException; import java.io.Serializable; import java.util.Collection; import java.util.Iterator; import java.util.Map; import java.util.Set; +import org.apache.hudi.common.util.RocksDBDAO; +import org.apache.hudi.exception.HoodieNotSupportedException; public final class RocksDBBasedMap implements Map { diff --git a/hoodie-common/src/main/java/com/uber/hoodie/common/util/collection/Triple.java b/hudi-common/src/main/java/org/apache/hudi/common/util/collection/Triple.java similarity index 99% rename from hoodie-common/src/main/java/com/uber/hoodie/common/util/collection/Triple.java rename to hudi-common/src/main/java/org/apache/hudi/common/util/collection/Triple.java index 3b1008da2..8a030e193 100644 --- a/hoodie-common/src/main/java/com/uber/hoodie/common/util/collection/Triple.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/util/collection/Triple.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package com.uber.hoodie.common.util.collection; +package org.apache.hudi.common.util.collection; import java.io.Serializable; import org.apache.commons.lang.builder.CompareToBuilder; diff --git a/hoodie-common/src/main/java/com/uber/hoodie/common/util/collection/io/storage/SizeAwareDataOutputStream.java b/hudi-common/src/main/java/org/apache/hudi/common/util/collection/io/storage/SizeAwareDataOutputStream.java similarity index 97% rename from hoodie-common/src/main/java/com/uber/hoodie/common/util/collection/io/storage/SizeAwareDataOutputStream.java rename to hudi-common/src/main/java/org/apache/hudi/common/util/collection/io/storage/SizeAwareDataOutputStream.java index ad801e090..631f02d9a 100644 --- a/hoodie-common/src/main/java/com/uber/hoodie/common/util/collection/io/storage/SizeAwareDataOutputStream.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/util/collection/io/storage/SizeAwareDataOutputStream.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package com.uber.hoodie.common.util.collection.io.storage; +package org.apache.hudi.common.util.collection.io.storage; import java.io.DataOutputStream; import java.io.FileOutputStream; diff --git a/hoodie-common/src/main/java/com/uber/hoodie/common/util/queue/BoundedInMemoryExecutor.java b/hudi-common/src/main/java/org/apache/hudi/common/util/queue/BoundedInMemoryExecutor.java similarity index 95% rename from hoodie-common/src/main/java/com/uber/hoodie/common/util/queue/BoundedInMemoryExecutor.java rename to hudi-common/src/main/java/org/apache/hudi/common/util/queue/BoundedInMemoryExecutor.java index fe53dca24..e0614f8a0 100644 --- a/hoodie-common/src/main/java/com/uber/hoodie/common/util/queue/BoundedInMemoryExecutor.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/util/queue/BoundedInMemoryExecutor.java @@ -17,12 +17,8 @@ */ -package com.uber.hoodie.common.util.queue; +package org.apache.hudi.common.util.queue; -import com.uber.hoodie.common.util.DefaultSizeEstimator; -import com.uber.hoodie.common.util.Option; -import com.uber.hoodie.common.util.SizeEstimator; -import com.uber.hoodie.exception.HoodieException; import java.util.Arrays; import java.util.List; import java.util.concurrent.CompletableFuture; @@ -33,6 +29,10 @@ import java.util.concurrent.Executors; import java.util.concurrent.Future; import java.util.function.Function; import java.util.stream.Collectors; +import org.apache.hudi.common.util.DefaultSizeEstimator; +import org.apache.hudi.common.util.Option; +import org.apache.hudi.common.util.SizeEstimator; +import org.apache.hudi.exception.HoodieException; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; diff --git a/hoodie-common/src/main/java/com/uber/hoodie/common/util/queue/BoundedInMemoryQueue.java b/hudi-common/src/main/java/org/apache/hudi/common/util/queue/BoundedInMemoryQueue.java similarity index 97% rename from hoodie-common/src/main/java/com/uber/hoodie/common/util/queue/BoundedInMemoryQueue.java rename to hudi-common/src/main/java/org/apache/hudi/common/util/queue/BoundedInMemoryQueue.java index a77edc63d..0cc142dea 100644 --- a/hoodie-common/src/main/java/com/uber/hoodie/common/util/queue/BoundedInMemoryQueue.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/util/queue/BoundedInMemoryQueue.java @@ -16,14 +16,10 @@ * limitations under the License. */ -package com.uber.hoodie.common.util.queue; +package org.apache.hudi.common.util.queue; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; -import com.uber.hoodie.common.util.DefaultSizeEstimator; -import com.uber.hoodie.common.util.Option; -import com.uber.hoodie.common.util.SizeEstimator; -import com.uber.hoodie.exception.HoodieException; import java.util.Iterator; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.Semaphore; @@ -32,6 +28,10 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; import java.util.function.Function; +import org.apache.hudi.common.util.DefaultSizeEstimator; +import org.apache.hudi.common.util.Option; +import org.apache.hudi.common.util.SizeEstimator; +import org.apache.hudi.exception.HoodieException; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; diff --git a/hoodie-common/src/main/java/com/uber/hoodie/common/util/queue/BoundedInMemoryQueueConsumer.java b/hudi-common/src/main/java/org/apache/hudi/common/util/queue/BoundedInMemoryQueueConsumer.java similarity index 97% rename from hoodie-common/src/main/java/com/uber/hoodie/common/util/queue/BoundedInMemoryQueueConsumer.java rename to hudi-common/src/main/java/org/apache/hudi/common/util/queue/BoundedInMemoryQueueConsumer.java index 3a752a692..fb7369782 100644 --- a/hoodie-common/src/main/java/com/uber/hoodie/common/util/queue/BoundedInMemoryQueueConsumer.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/util/queue/BoundedInMemoryQueueConsumer.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package com.uber.hoodie.common.util.queue; +package org.apache.hudi.common.util.queue; import java.util.Iterator; diff --git a/hoodie-common/src/main/java/com/uber/hoodie/common/util/queue/BoundedInMemoryQueueProducer.java b/hudi-common/src/main/java/org/apache/hudi/common/util/queue/BoundedInMemoryQueueProducer.java similarity index 96% rename from hoodie-common/src/main/java/com/uber/hoodie/common/util/queue/BoundedInMemoryQueueProducer.java rename to hudi-common/src/main/java/org/apache/hudi/common/util/queue/BoundedInMemoryQueueProducer.java index 986ca0f89..41ec245e9 100644 --- a/hoodie-common/src/main/java/com/uber/hoodie/common/util/queue/BoundedInMemoryQueueProducer.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/util/queue/BoundedInMemoryQueueProducer.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package com.uber.hoodie.common.util.queue; +package org.apache.hudi.common.util.queue; /** * Producer for BoundedInMemoryQueue. Memory Bounded Buffer supports diff --git a/hoodie-common/src/main/java/com/uber/hoodie/common/util/queue/FunctionBasedQueueProducer.java b/hudi-common/src/main/java/org/apache/hudi/common/util/queue/FunctionBasedQueueProducer.java similarity index 97% rename from hoodie-common/src/main/java/com/uber/hoodie/common/util/queue/FunctionBasedQueueProducer.java rename to hudi-common/src/main/java/org/apache/hudi/common/util/queue/FunctionBasedQueueProducer.java index cd226dd5b..175dbf241 100644 --- a/hoodie-common/src/main/java/com/uber/hoodie/common/util/queue/FunctionBasedQueueProducer.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/util/queue/FunctionBasedQueueProducer.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package com.uber.hoodie.common.util.queue; +package org.apache.hudi.common.util.queue; import java.util.function.Function; import org.apache.log4j.LogManager; diff --git a/hoodie-common/src/main/java/com/uber/hoodie/common/util/queue/IteratorBasedQueueProducer.java b/hudi-common/src/main/java/org/apache/hudi/common/util/queue/IteratorBasedQueueProducer.java similarity index 97% rename from hoodie-common/src/main/java/com/uber/hoodie/common/util/queue/IteratorBasedQueueProducer.java rename to hudi-common/src/main/java/org/apache/hudi/common/util/queue/IteratorBasedQueueProducer.java index 289272be0..5d52012d6 100644 --- a/hoodie-common/src/main/java/com/uber/hoodie/common/util/queue/IteratorBasedQueueProducer.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/util/queue/IteratorBasedQueueProducer.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package com.uber.hoodie.common.util.queue; +package org.apache.hudi.common.util.queue; import java.util.Iterator; import org.apache.log4j.LogManager; diff --git a/hoodie-common/src/main/java/com/uber/hoodie/config/DefaultHoodieConfig.java b/hudi-common/src/main/java/org/apache/hudi/config/DefaultHoodieConfig.java similarity index 98% rename from hoodie-common/src/main/java/com/uber/hoodie/config/DefaultHoodieConfig.java rename to hudi-common/src/main/java/org/apache/hudi/config/DefaultHoodieConfig.java index 45948d545..406606fe4 100644 --- a/hoodie-common/src/main/java/com/uber/hoodie/config/DefaultHoodieConfig.java +++ b/hudi-common/src/main/java/org/apache/hudi/config/DefaultHoodieConfig.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package com.uber.hoodie.config; +package org.apache.hudi.config; import java.io.Serializable; import java.util.Properties; diff --git a/hoodie-common/src/main/java/com/uber/hoodie/exception/CorruptedLogFileException.java b/hudi-common/src/main/java/org/apache/hudi/exception/CorruptedLogFileException.java similarity index 96% rename from hoodie-common/src/main/java/com/uber/hoodie/exception/CorruptedLogFileException.java rename to hudi-common/src/main/java/org/apache/hudi/exception/CorruptedLogFileException.java index c92f23d91..5a7665920 100644 --- a/hoodie-common/src/main/java/com/uber/hoodie/exception/CorruptedLogFileException.java +++ b/hudi-common/src/main/java/org/apache/hudi/exception/CorruptedLogFileException.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package com.uber.hoodie.exception; +package org.apache.hudi.exception; public class CorruptedLogFileException extends HoodieException { diff --git a/hoodie-common/src/main/java/com/uber/hoodie/exception/DatasetNotFoundException.java b/hudi-common/src/main/java/org/apache/hudi/exception/DatasetNotFoundException.java similarity index 98% rename from hoodie-common/src/main/java/com/uber/hoodie/exception/DatasetNotFoundException.java rename to hudi-common/src/main/java/org/apache/hudi/exception/DatasetNotFoundException.java index e0e862800..87dd549e1 100644 --- a/hoodie-common/src/main/java/com/uber/hoodie/exception/DatasetNotFoundException.java +++ b/hudi-common/src/main/java/org/apache/hudi/exception/DatasetNotFoundException.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package com.uber.hoodie.exception; +package org.apache.hudi.exception; import java.io.IOException; import org.apache.hadoop.fs.FileSystem; diff --git a/hoodie-common/src/main/java/com/uber/hoodie/exception/HoodieCorruptedDataException.java b/hudi-common/src/main/java/org/apache/hudi/exception/HoodieCorruptedDataException.java similarity index 97% rename from hoodie-common/src/main/java/com/uber/hoodie/exception/HoodieCorruptedDataException.java rename to hudi-common/src/main/java/org/apache/hudi/exception/HoodieCorruptedDataException.java index 965c21cb2..9cd48a5b5 100644 --- a/hoodie-common/src/main/java/com/uber/hoodie/exception/HoodieCorruptedDataException.java +++ b/hudi-common/src/main/java/org/apache/hudi/exception/HoodieCorruptedDataException.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package com.uber.hoodie.exception; +package org.apache.hudi.exception; /** *

Exception thrown when any data corruption happens when reading/writing from temporary disk diff --git a/hoodie-common/src/main/java/com/uber/hoodie/exception/HoodieException.java b/hudi-common/src/main/java/org/apache/hudi/exception/HoodieException.java similarity index 97% rename from hoodie-common/src/main/java/com/uber/hoodie/exception/HoodieException.java rename to hudi-common/src/main/java/org/apache/hudi/exception/HoodieException.java index 711b7350c..580f804d0 100644 --- a/hoodie-common/src/main/java/com/uber/hoodie/exception/HoodieException.java +++ b/hudi-common/src/main/java/org/apache/hudi/exception/HoodieException.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package com.uber.hoodie.exception; +package org.apache.hudi.exception; import java.io.Serializable; diff --git a/hoodie-common/src/main/java/com/uber/hoodie/exception/HoodieIOException.java b/hudi-common/src/main/java/org/apache/hudi/exception/HoodieIOException.java similarity index 97% rename from hoodie-common/src/main/java/com/uber/hoodie/exception/HoodieIOException.java rename to hudi-common/src/main/java/org/apache/hudi/exception/HoodieIOException.java index 6d1043bab..438e0dc9c 100644 --- a/hoodie-common/src/main/java/com/uber/hoodie/exception/HoodieIOException.java +++ b/hudi-common/src/main/java/org/apache/hudi/exception/HoodieIOException.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package com.uber.hoodie.exception; +package org.apache.hudi.exception; import java.io.IOException; diff --git a/hoodie-common/src/main/java/com/uber/hoodie/exception/HoodieIndexException.java b/hudi-common/src/main/java/org/apache/hudi/exception/HoodieIndexException.java similarity index 96% rename from hoodie-common/src/main/java/com/uber/hoodie/exception/HoodieIndexException.java rename to hudi-common/src/main/java/org/apache/hudi/exception/HoodieIndexException.java index 7a31a2ece..bc923d506 100644 --- a/hoodie-common/src/main/java/com/uber/hoodie/exception/HoodieIndexException.java +++ b/hudi-common/src/main/java/org/apache/hudi/exception/HoodieIndexException.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package com.uber.hoodie.exception; +package org.apache.hudi.exception; /** *

Exception thrown for HoodieIndex related errors.

diff --git a/hoodie-common/src/main/java/com/uber/hoodie/exception/HoodieNotSupportedException.java b/hudi-common/src/main/java/org/apache/hudi/exception/HoodieNotSupportedException.java similarity index 96% rename from hoodie-common/src/main/java/com/uber/hoodie/exception/HoodieNotSupportedException.java rename to hudi-common/src/main/java/org/apache/hudi/exception/HoodieNotSupportedException.java index 750f13e69..68d2142a5 100644 --- a/hoodie-common/src/main/java/com/uber/hoodie/exception/HoodieNotSupportedException.java +++ b/hudi-common/src/main/java/org/apache/hudi/exception/HoodieNotSupportedException.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package com.uber.hoodie.exception; +package org.apache.hudi.exception; public class HoodieNotSupportedException extends HoodieException { diff --git a/hoodie-common/src/main/java/com/uber/hoodie/exception/HoodieRecordMissingException.java b/hudi-common/src/main/java/org/apache/hudi/exception/HoodieRecordMissingException.java similarity index 94% rename from hoodie-common/src/main/java/com/uber/hoodie/exception/HoodieRecordMissingException.java rename to hudi-common/src/main/java/org/apache/hudi/exception/HoodieRecordMissingException.java index e656849ba..1bc6c5716 100644 --- a/hoodie-common/src/main/java/com/uber/hoodie/exception/HoodieRecordMissingException.java +++ b/hudi-common/src/main/java/org/apache/hudi/exception/HoodieRecordMissingException.java @@ -16,9 +16,9 @@ * limitations under the License. */ -package com.uber.hoodie.exception; +package org.apache.hudi.exception; -import com.uber.hoodie.common.model.HoodieRecord; +import org.apache.hudi.common.model.HoodieRecord; /** *

Exception throws when indexing fails to locate the hoodie record. HoodieRecord current diff --git a/hoodie-common/src/main/java/com/uber/hoodie/exception/HoodieRemoteException.java b/hudi-common/src/main/java/org/apache/hudi/exception/HoodieRemoteException.java similarity index 96% rename from hoodie-common/src/main/java/com/uber/hoodie/exception/HoodieRemoteException.java rename to hudi-common/src/main/java/org/apache/hudi/exception/HoodieRemoteException.java index ed4f4ae76..c9c5c0916 100644 --- a/hoodie-common/src/main/java/com/uber/hoodie/exception/HoodieRemoteException.java +++ b/hudi-common/src/main/java/org/apache/hudi/exception/HoodieRemoteException.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package com.uber.hoodie.exception; +package org.apache.hudi.exception; import java.io.IOException; diff --git a/hoodie-common/src/main/java/com/uber/hoodie/exception/HoodieSerializationException.java b/hudi-common/src/main/java/org/apache/hudi/exception/HoodieSerializationException.java similarity index 97% rename from hoodie-common/src/main/java/com/uber/hoodie/exception/HoodieSerializationException.java rename to hudi-common/src/main/java/org/apache/hudi/exception/HoodieSerializationException.java index fb8bee6b0..5dab27cbe 100644 --- a/hoodie-common/src/main/java/com/uber/hoodie/exception/HoodieSerializationException.java +++ b/hudi-common/src/main/java/org/apache/hudi/exception/HoodieSerializationException.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package com.uber.hoodie.exception; +package org.apache.hudi.exception; import java.io.IOException; diff --git a/hoodie-common/src/main/java/com/uber/hoodie/exception/InvalidDatasetException.java b/hudi-common/src/main/java/org/apache/hudi/exception/InvalidDatasetException.java similarity index 97% rename from hoodie-common/src/main/java/com/uber/hoodie/exception/InvalidDatasetException.java rename to hudi-common/src/main/java/org/apache/hudi/exception/InvalidDatasetException.java index 80b002a07..943aa1594 100644 --- a/hoodie-common/src/main/java/com/uber/hoodie/exception/InvalidDatasetException.java +++ b/hudi-common/src/main/java/org/apache/hudi/exception/InvalidDatasetException.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package com.uber.hoodie.exception; +package org.apache.hudi.exception; /** *

Exception thrown to indicate that a hoodie dataset is invalid

diff --git a/hoodie-common/src/main/java/com/uber/hoodie/exception/InvalidHoodiePathException.java b/hudi-common/src/main/java/org/apache/hudi/exception/InvalidHoodiePathException.java similarity index 96% rename from hoodie-common/src/main/java/com/uber/hoodie/exception/InvalidHoodiePathException.java rename to hudi-common/src/main/java/org/apache/hudi/exception/InvalidHoodiePathException.java index 856189bbf..9e6e75e59 100644 --- a/hoodie-common/src/main/java/com/uber/hoodie/exception/InvalidHoodiePathException.java +++ b/hudi-common/src/main/java/org/apache/hudi/exception/InvalidHoodiePathException.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package com.uber.hoodie.exception; +package org.apache.hudi.exception; import org.apache.hadoop.fs.Path; diff --git a/hoodie-common/src/main/java/com/uber/hoodie/exception/MetadataNotFoundException.java b/hudi-common/src/main/java/org/apache/hudi/exception/MetadataNotFoundException.java similarity index 96% rename from hoodie-common/src/main/java/com/uber/hoodie/exception/MetadataNotFoundException.java rename to hudi-common/src/main/java/org/apache/hudi/exception/MetadataNotFoundException.java index 63aa65f13..7d02c624c 100644 --- a/hoodie-common/src/main/java/com/uber/hoodie/exception/MetadataNotFoundException.java +++ b/hudi-common/src/main/java/org/apache/hudi/exception/MetadataNotFoundException.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package com.uber.hoodie.exception; +package org.apache.hudi.exception; /** * Thrown when expected metadata is not found diff --git a/hoodie-common/src/main/java/com/uber/hoodie/exception/SchemaCompatabilityException.java b/hudi-common/src/main/java/org/apache/hudi/exception/SchemaCompatabilityException.java similarity index 96% rename from hoodie-common/src/main/java/com/uber/hoodie/exception/SchemaCompatabilityException.java rename to hudi-common/src/main/java/org/apache/hudi/exception/SchemaCompatabilityException.java index d6ee8542f..33efe11cc 100644 --- a/hoodie-common/src/main/java/com/uber/hoodie/exception/SchemaCompatabilityException.java +++ b/hudi-common/src/main/java/org/apache/hudi/exception/SchemaCompatabilityException.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package com.uber.hoodie.exception; +package org.apache.hudi.exception; public class SchemaCompatabilityException extends HoodieException { diff --git a/hoodie-common/src/main/resources/META-INF/LICENSE.txt b/hudi-common/src/main/resources/META-INF/LICENSE.txt similarity index 100% rename from hoodie-common/src/main/resources/META-INF/LICENSE.txt rename to hudi-common/src/main/resources/META-INF/LICENSE.txt diff --git a/hoodie-common/src/main/resources/META-INF/NOTICE.txt b/hudi-common/src/main/resources/META-INF/NOTICE.txt similarity index 100% rename from hoodie-common/src/main/resources/META-INF/NOTICE.txt rename to hudi-common/src/main/resources/META-INF/NOTICE.txt diff --git a/hoodie-common/src/test/java/com/uber/hoodie/common/TestBloomFilter.java b/hudi-common/src/test/java/org/apache/hudi/common/TestBloomFilter.java similarity index 97% rename from hoodie-common/src/test/java/com/uber/hoodie/common/TestBloomFilter.java rename to hudi-common/src/test/java/org/apache/hudi/common/TestBloomFilter.java index 08274e289..76e980568 100644 --- a/hoodie-common/src/test/java/com/uber/hoodie/common/TestBloomFilter.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/TestBloomFilter.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package com.uber.hoodie.common; +package org.apache.hudi.common; import java.io.IOException; import org.junit.Test; diff --git a/hoodie-common/src/test/java/com/uber/hoodie/common/minicluster/HdfsTestService.java b/hudi-common/src/test/java/org/apache/hudi/common/minicluster/HdfsTestService.java similarity index 98% rename from hoodie-common/src/test/java/com/uber/hoodie/common/minicluster/HdfsTestService.java rename to hudi-common/src/test/java/org/apache/hudi/common/minicluster/HdfsTestService.java index c9d92c280..5408189fe 100644 --- a/hoodie-common/src/test/java/com/uber/hoodie/common/minicluster/HdfsTestService.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/minicluster/HdfsTestService.java @@ -16,11 +16,10 @@ * limitations under the License. */ -package com.uber.hoodie.common.minicluster; +package org.apache.hudi.common.minicluster; import com.google.common.base.Preconditions; import com.google.common.io.Files; -import com.uber.hoodie.common.model.HoodieTestUtils; import java.io.File; import java.io.IOException; import org.apache.commons.io.FileUtils; @@ -28,6 +27,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.hudi.common.model.HoodieTestUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/hoodie-common/src/test/java/com/uber/hoodie/common/minicluster/MiniClusterUtil.java b/hudi-common/src/test/java/org/apache/hudi/common/minicluster/MiniClusterUtil.java similarity index 97% rename from hoodie-common/src/test/java/com/uber/hoodie/common/minicluster/MiniClusterUtil.java rename to hudi-common/src/test/java/org/apache/hudi/common/minicluster/MiniClusterUtil.java index 81b344c8e..a671d1db5 100644 --- a/hoodie-common/src/test/java/com/uber/hoodie/common/minicluster/MiniClusterUtil.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/minicluster/MiniClusterUtil.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package com.uber.hoodie.common.minicluster; +package org.apache.hudi.common.minicluster; import java.io.IOException; import org.apache.hadoop.conf.Configuration; diff --git a/hoodie-common/src/test/java/com/uber/hoodie/common/minicluster/ZookeeperTestService.java b/hudi-common/src/test/java/org/apache/hudi/common/minicluster/ZookeeperTestService.java similarity index 99% rename from hoodie-common/src/test/java/com/uber/hoodie/common/minicluster/ZookeeperTestService.java rename to hudi-common/src/test/java/org/apache/hudi/common/minicluster/ZookeeperTestService.java index 79ebb2918..1a01012a2 100644 --- a/hoodie-common/src/test/java/com/uber/hoodie/common/minicluster/ZookeeperTestService.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/minicluster/ZookeeperTestService.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package com.uber.hoodie.common.minicluster; +package org.apache.hudi.common.minicluster; import com.google.common.base.Preconditions; import com.google.common.io.Files; diff --git a/hoodie-common/src/test/java/com/uber/hoodie/common/model/AvroBinaryTestPayload.java b/hudi-common/src/test/java/org/apache/hudi/common/model/AvroBinaryTestPayload.java similarity index 90% rename from hoodie-common/src/test/java/com/uber/hoodie/common/model/AvroBinaryTestPayload.java rename to hudi-common/src/test/java/org/apache/hudi/common/model/AvroBinaryTestPayload.java index 1908bf906..bd3c3e0e9 100644 --- a/hoodie-common/src/test/java/com/uber/hoodie/common/model/AvroBinaryTestPayload.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/model/AvroBinaryTestPayload.java @@ -16,15 +16,15 @@ * limitations under the License. */ -package com.uber.hoodie.common.model; +package org.apache.hudi.common.model; -import com.uber.hoodie.common.util.HoodieAvroUtils; -import com.uber.hoodie.common.util.Option; -import com.uber.hoodie.exception.HoodieIOException; import java.io.IOException; import org.apache.avro.Schema; import org.apache.avro.generic.GenericRecord; import org.apache.avro.generic.IndexedRecord; +import org.apache.hudi.common.util.HoodieAvroUtils; +import org.apache.hudi.common.util.Option; +import org.apache.hudi.exception.HoodieIOException; public class AvroBinaryTestPayload implements HoodieRecordPayload { diff --git a/hoodie-common/src/test/java/com/uber/hoodie/common/model/HoodieTestUtils.java b/hudi-common/src/test/java/org/apache/hudi/common/model/HoodieTestUtils.java similarity index 93% rename from hoodie-common/src/test/java/com/uber/hoodie/common/model/HoodieTestUtils.java rename to hudi-common/src/test/java/org/apache/hudi/common/model/HoodieTestUtils.java index 7a00606d8..056c06970 100644 --- a/hoodie-common/src/test/java/com/uber/hoodie/common/model/HoodieTestUtils.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/model/HoodieTestUtils.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package com.uber.hoodie.common.model; +package org.apache.hudi.common.model; import static org.junit.Assert.assertEquals; import static org.junit.Assert.fail; @@ -27,26 +27,6 @@ import com.esotericsoftware.kryo.io.Output; import com.esotericsoftware.kryo.serializers.JavaSerializer; import com.google.common.collect.Lists; import com.google.common.collect.Maps; -import com.uber.hoodie.avro.model.HoodieCleanMetadata; -import com.uber.hoodie.avro.model.HoodieCompactionPlan; -import com.uber.hoodie.common.HoodieCleanStat; -import com.uber.hoodie.common.model.HoodieWriteStat.RuntimeStats; -import com.uber.hoodie.common.table.HoodieTableConfig; -import com.uber.hoodie.common.table.HoodieTableMetaClient; -import com.uber.hoodie.common.table.HoodieTimeline; -import com.uber.hoodie.common.table.log.HoodieLogFormat; -import com.uber.hoodie.common.table.log.HoodieLogFormat.Writer; -import com.uber.hoodie.common.table.log.block.HoodieAvroDataBlock; -import com.uber.hoodie.common.table.log.block.HoodieLogBlock; -import com.uber.hoodie.common.table.timeline.HoodieActiveTimeline; -import com.uber.hoodie.common.table.timeline.HoodieInstant; -import com.uber.hoodie.common.table.timeline.HoodieInstant.State; -import com.uber.hoodie.common.util.AvroUtils; -import com.uber.hoodie.common.util.CompactionUtils; -import com.uber.hoodie.common.util.FSUtils; -import com.uber.hoodie.common.util.HoodieAvroUtils; -import com.uber.hoodie.common.util.Option; -import com.uber.hoodie.common.util.collection.Pair; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.File; @@ -76,6 +56,26 @@ import org.apache.hadoop.fs.LocatedFileStatus; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.RemoteIterator; import org.apache.hadoop.util.StringUtils; +import org.apache.hudi.avro.model.HoodieCleanMetadata; +import org.apache.hudi.avro.model.HoodieCompactionPlan; +import org.apache.hudi.common.HoodieCleanStat; +import org.apache.hudi.common.model.HoodieWriteStat.RuntimeStats; +import org.apache.hudi.common.table.HoodieTableConfig; +import org.apache.hudi.common.table.HoodieTableMetaClient; +import org.apache.hudi.common.table.HoodieTimeline; +import org.apache.hudi.common.table.log.HoodieLogFormat; +import org.apache.hudi.common.table.log.HoodieLogFormat.Writer; +import org.apache.hudi.common.table.log.block.HoodieAvroDataBlock; +import org.apache.hudi.common.table.log.block.HoodieLogBlock; +import org.apache.hudi.common.table.timeline.HoodieActiveTimeline; +import org.apache.hudi.common.table.timeline.HoodieInstant; +import org.apache.hudi.common.table.timeline.HoodieInstant.State; +import org.apache.hudi.common.util.AvroUtils; +import org.apache.hudi.common.util.CompactionUtils; +import org.apache.hudi.common.util.FSUtils; +import org.apache.hudi.common.util.HoodieAvroUtils; +import org.apache.hudi.common.util.Option; +import org.apache.hudi.common.util.collection.Pair; public class HoodieTestUtils { diff --git a/hoodie-common/src/test/java/com/uber/hoodie/common/model/TestHoodieCommitMetadata.java b/hudi-common/src/test/java/org/apache/hudi/common/model/TestHoodieCommitMetadata.java similarity index 98% rename from hoodie-common/src/test/java/com/uber/hoodie/common/model/TestHoodieCommitMetadata.java rename to hudi-common/src/test/java/org/apache/hudi/common/model/TestHoodieCommitMetadata.java index c317261b2..9c755fcbe 100644 --- a/hoodie-common/src/test/java/com/uber/hoodie/common/model/TestHoodieCommitMetadata.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/model/TestHoodieCommitMetadata.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package com.uber.hoodie.common.model; +package org.apache.hudi.common.model; import java.util.List; import org.junit.Assert; diff --git a/hoodie-common/src/test/java/com/uber/hoodie/common/model/TestHoodieWriteStat.java b/hudi-common/src/test/java/org/apache/hudi/common/model/TestHoodieWriteStat.java similarity index 93% rename from hoodie-common/src/test/java/com/uber/hoodie/common/model/TestHoodieWriteStat.java rename to hudi-common/src/test/java/org/apache/hudi/common/model/TestHoodieWriteStat.java index 420e5d466..5c460f9c4 100644 --- a/hoodie-common/src/test/java/com/uber/hoodie/common/model/TestHoodieWriteStat.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/model/TestHoodieWriteStat.java @@ -16,17 +16,17 @@ * limitations under the License. */ -package com.uber.hoodie.common.model; +package org.apache.hudi.common.model; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNull; -import com.uber.hoodie.common.table.HoodieTableMetaClient; -import com.uber.hoodie.common.util.FSUtils; import java.text.SimpleDateFormat; import java.util.Date; import java.util.UUID; import org.apache.hadoop.fs.Path; +import org.apache.hudi.common.table.HoodieTableMetaClient; +import org.apache.hudi.common.util.FSUtils; import org.junit.Test; public class TestHoodieWriteStat { diff --git a/hoodie-common/src/test/java/com/uber/hoodie/common/storage/TestStorageSchemes.java b/hudi-common/src/test/java/org/apache/hudi/common/storage/TestStorageSchemes.java similarity index 97% rename from hoodie-common/src/test/java/com/uber/hoodie/common/storage/TestStorageSchemes.java rename to hudi-common/src/test/java/org/apache/hudi/common/storage/TestStorageSchemes.java index 635dfae4f..fa4bd6e2c 100644 --- a/hoodie-common/src/test/java/com/uber/hoodie/common/storage/TestStorageSchemes.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/storage/TestStorageSchemes.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package com.uber.hoodie.common.storage; +package org.apache.hudi.common.storage; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; diff --git a/hoodie-common/src/test/java/com/uber/hoodie/common/table/HoodieTableMetaClientTest.java b/hudi-common/src/test/java/org/apache/hudi/common/table/HoodieTableMetaClientTest.java similarity index 94% rename from hoodie-common/src/test/java/com/uber/hoodie/common/table/HoodieTableMetaClientTest.java rename to hudi-common/src/test/java/org/apache/hudi/common/table/HoodieTableMetaClientTest.java index e30bdcc34..df72a7ce2 100644 --- a/hoodie-common/src/test/java/com/uber/hoodie/common/table/HoodieTableMetaClientTest.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/table/HoodieTableMetaClientTest.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package com.uber.hoodie.common.table; +package org.apache.hudi.common.table; import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; @@ -25,17 +25,17 @@ import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; import com.google.common.collect.Lists; -import com.uber.hoodie.common.model.HoodieTestUtils; -import com.uber.hoodie.common.table.timeline.HoodieActiveTimeline; -import com.uber.hoodie.common.table.timeline.HoodieArchivedTimeline; -import com.uber.hoodie.common.table.timeline.HoodieInstant; -import com.uber.hoodie.common.util.Option; import java.io.IOException; import java.util.stream.Collectors; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.io.SequenceFile; import org.apache.hadoop.io.Text; +import org.apache.hudi.common.model.HoodieTestUtils; +import org.apache.hudi.common.table.timeline.HoodieActiveTimeline; +import org.apache.hudi.common.table.timeline.HoodieArchivedTimeline; +import org.apache.hudi.common.table.timeline.HoodieInstant; +import org.apache.hudi.common.util.Option; import org.junit.Before; import org.junit.Rule; import org.junit.Test; diff --git a/hoodie-common/src/test/java/com/uber/hoodie/common/table/log/HoodieLogFormatAppendFailureTest.java b/hudi-common/src/test/java/org/apache/hudi/common/table/log/HoodieLogFormatAppendFailureTest.java similarity index 92% rename from hoodie-common/src/test/java/com/uber/hoodie/common/table/log/HoodieLogFormatAppendFailureTest.java rename to hudi-common/src/test/java/org/apache/hudi/common/table/log/HoodieLogFormatAppendFailureTest.java index 4a6f23358..b7d592887 100644 --- a/hoodie-common/src/test/java/com/uber/hoodie/common/table/log/HoodieLogFormatAppendFailureTest.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/table/log/HoodieLogFormatAppendFailureTest.java @@ -16,17 +16,11 @@ * limitations under the License. */ -package com.uber.hoodie.common.table.log; +package org.apache.hudi.common.table.log; -import static com.uber.hoodie.common.util.SchemaTestUtil.getSimpleSchema; +import static org.apache.hudi.common.util.SchemaTestUtil.getSimpleSchema; import com.google.common.collect.Maps; -import com.uber.hoodie.common.minicluster.MiniClusterUtil; -import com.uber.hoodie.common.model.HoodieArchivedLogFile; -import com.uber.hoodie.common.table.log.HoodieLogFormat.Writer; -import com.uber.hoodie.common.table.log.block.HoodieAvroDataBlock; -import com.uber.hoodie.common.table.log.block.HoodieLogBlock; -import com.uber.hoodie.common.util.SchemaTestUtil; import java.io.File; import java.io.IOException; import java.net.URISyntaxException; @@ -45,6 +39,12 @@ import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; import org.apache.hadoop.hdfs.protocol.LocatedBlocks; import org.apache.hadoop.hdfs.server.datanode.DataNode; +import org.apache.hudi.common.minicluster.MiniClusterUtil; +import org.apache.hudi.common.model.HoodieArchivedLogFile; +import org.apache.hudi.common.table.log.HoodieLogFormat.Writer; +import org.apache.hudi.common.table.log.block.HoodieAvroDataBlock; +import org.apache.hudi.common.table.log.block.HoodieLogBlock; +import org.apache.hudi.common.util.SchemaTestUtil; import org.junit.AfterClass; import org.junit.Assert; import org.junit.BeforeClass; diff --git a/hoodie-common/src/test/java/com/uber/hoodie/common/table/log/HoodieLogFormatTest.java b/hudi-common/src/test/java/org/apache/hudi/common/table/log/HoodieLogFormatTest.java similarity index 98% rename from hoodie-common/src/test/java/com/uber/hoodie/common/table/log/HoodieLogFormatTest.java rename to hudi-common/src/test/java/org/apache/hudi/common/table/log/HoodieLogFormatTest.java index 4b4a3e078..4df738f5f 100644 --- a/hoodie-common/src/test/java/com/uber/hoodie/common/table/log/HoodieLogFormatTest.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/table/log/HoodieLogFormatTest.java @@ -16,9 +16,9 @@ * limitations under the License. */ -package com.uber.hoodie.common.table.log; +package org.apache.hudi.common.table.log; -import static com.uber.hoodie.common.util.SchemaTestUtil.getSimpleSchema; +import static org.apache.hudi.common.util.SchemaTestUtil.getSimpleSchema; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; @@ -26,26 +26,6 @@ import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import com.google.common.collect.Maps; -import com.uber.hoodie.common.minicluster.MiniClusterUtil; -import com.uber.hoodie.common.model.HoodieArchivedLogFile; -import com.uber.hoodie.common.model.HoodieKey; -import com.uber.hoodie.common.model.HoodieLogFile; -import com.uber.hoodie.common.model.HoodieRecord; -import com.uber.hoodie.common.model.HoodieTableType; -import com.uber.hoodie.common.model.HoodieTestUtils; -import com.uber.hoodie.common.table.log.HoodieLogFormat.Reader; -import com.uber.hoodie.common.table.log.HoodieLogFormat.Writer; -import com.uber.hoodie.common.table.log.block.HoodieAvroDataBlock; -import com.uber.hoodie.common.table.log.block.HoodieCommandBlock; -import com.uber.hoodie.common.table.log.block.HoodieCorruptBlock; -import com.uber.hoodie.common.table.log.block.HoodieDeleteBlock; -import com.uber.hoodie.common.table.log.block.HoodieLogBlock; -import com.uber.hoodie.common.table.log.block.HoodieLogBlock.HeaderMetadataType; -import com.uber.hoodie.common.table.log.block.HoodieLogBlock.HoodieLogBlockType; -import com.uber.hoodie.common.util.FSUtils; -import com.uber.hoodie.common.util.HoodieAvroUtils; -import com.uber.hoodie.common.util.SchemaTestUtil; -import com.uber.hoodie.exception.CorruptedLogFileException; import java.io.IOException; import java.io.UncheckedIOException; import java.net.URISyntaxException; @@ -65,6 +45,26 @@ import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hudi.common.minicluster.MiniClusterUtil; +import org.apache.hudi.common.model.HoodieArchivedLogFile; +import org.apache.hudi.common.model.HoodieKey; +import org.apache.hudi.common.model.HoodieLogFile; +import org.apache.hudi.common.model.HoodieRecord; +import org.apache.hudi.common.model.HoodieTableType; +import org.apache.hudi.common.model.HoodieTestUtils; +import org.apache.hudi.common.table.log.HoodieLogFormat.Reader; +import org.apache.hudi.common.table.log.HoodieLogFormat.Writer; +import org.apache.hudi.common.table.log.block.HoodieAvroDataBlock; +import org.apache.hudi.common.table.log.block.HoodieCommandBlock; +import org.apache.hudi.common.table.log.block.HoodieCorruptBlock; +import org.apache.hudi.common.table.log.block.HoodieDeleteBlock; +import org.apache.hudi.common.table.log.block.HoodieLogBlock; +import org.apache.hudi.common.table.log.block.HoodieLogBlock.HeaderMetadataType; +import org.apache.hudi.common.table.log.block.HoodieLogBlock.HoodieLogBlockType; +import org.apache.hudi.common.util.FSUtils; +import org.apache.hudi.common.util.HoodieAvroUtils; +import org.apache.hudi.common.util.SchemaTestUtil; +import org.apache.hudi.exception.CorruptedLogFileException; import org.junit.After; import org.junit.AfterClass; import org.junit.Before; diff --git a/hoodie-common/src/test/java/com/uber/hoodie/common/table/string/HoodieActiveTimelineTest.java b/hudi-common/src/test/java/org/apache/hudi/common/table/string/HoodieActiveTimelineTest.java similarity index 94% rename from hoodie-common/src/test/java/com/uber/hoodie/common/table/string/HoodieActiveTimelineTest.java rename to hudi-common/src/test/java/org/apache/hudi/common/table/string/HoodieActiveTimelineTest.java index 2f9574abf..cb26079a8 100644 --- a/hoodie-common/src/test/java/com/uber/hoodie/common/table/string/HoodieActiveTimelineTest.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/table/string/HoodieActiveTimelineTest.java @@ -16,20 +16,20 @@ * limitations under the License. */ -package com.uber.hoodie.common.table.string; +package org.apache.hudi.common.table.string; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; -import com.uber.hoodie.common.model.HoodieTestUtils; -import com.uber.hoodie.common.table.HoodieTableMetaClient; -import com.uber.hoodie.common.table.HoodieTimeline; -import com.uber.hoodie.common.table.timeline.HoodieActiveTimeline; -import com.uber.hoodie.common.table.timeline.HoodieInstant; -import com.uber.hoodie.common.util.Option; import java.io.IOException; import java.util.stream.Stream; +import org.apache.hudi.common.model.HoodieTestUtils; +import org.apache.hudi.common.table.HoodieTableMetaClient; +import org.apache.hudi.common.table.HoodieTimeline; +import org.apache.hudi.common.table.timeline.HoodieActiveTimeline; +import org.apache.hudi.common.table.timeline.HoodieInstant; +import org.apache.hudi.common.util.Option; import org.junit.Before; import org.junit.Rule; import org.junit.Test; diff --git a/hoodie-common/src/test/java/com/uber/hoodie/common/table/string/MockHoodieTimeline.java b/hudi-common/src/test/java/org/apache/hudi/common/table/string/MockHoodieTimeline.java similarity index 87% rename from hoodie-common/src/test/java/com/uber/hoodie/common/table/string/MockHoodieTimeline.java rename to hudi-common/src/test/java/org/apache/hudi/common/table/string/MockHoodieTimeline.java index d8ed35268..e30b61a4a 100644 --- a/hoodie-common/src/test/java/com/uber/hoodie/common/table/string/MockHoodieTimeline.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/table/string/MockHoodieTimeline.java @@ -16,16 +16,16 @@ * limitations under the License. */ -package com.uber.hoodie.common.table.string; +package org.apache.hudi.common.table.string; -import com.uber.hoodie.common.table.HoodieTimeline; -import com.uber.hoodie.common.table.timeline.HoodieActiveTimeline; -import com.uber.hoodie.common.table.timeline.HoodieInstant; import java.io.IOException; import java.util.Comparator; import java.util.function.Function; import java.util.stream.Collectors; import java.util.stream.Stream; +import org.apache.hudi.common.table.HoodieTimeline; +import org.apache.hudi.common.table.timeline.HoodieActiveTimeline; +import org.apache.hudi.common.table.timeline.HoodieInstant; public class MockHoodieTimeline extends HoodieActiveTimeline { diff --git a/hoodie-common/src/test/java/com/uber/hoodie/common/table/view/HoodieTableFileSystemViewTest.java b/hudi-common/src/test/java/org/apache/hudi/common/table/view/HoodieTableFileSystemViewTest.java similarity index 98% rename from hoodie-common/src/test/java/com/uber/hoodie/common/table/view/HoodieTableFileSystemViewTest.java rename to hudi-common/src/test/java/org/apache/hudi/common/table/view/HoodieTableFileSystemViewTest.java index 0d14d8e5f..35ddd92da 100644 --- a/hoodie-common/src/test/java/com/uber/hoodie/common/table/view/HoodieTableFileSystemViewTest.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/table/view/HoodieTableFileSystemViewTest.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package com.uber.hoodie.common.table.view; +package org.apache.hudi.common.table.view; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -24,27 +24,6 @@ import static org.junit.Assert.assertTrue; import com.google.common.collect.Lists; import com.google.common.collect.Sets; -import com.uber.hoodie.avro.model.HoodieCompactionPlan; -import com.uber.hoodie.common.model.CompactionOperation; -import com.uber.hoodie.common.model.FileSlice; -import com.uber.hoodie.common.model.HoodieDataFile; -import com.uber.hoodie.common.model.HoodieFileGroup; -import com.uber.hoodie.common.model.HoodieFileGroupId; -import com.uber.hoodie.common.model.HoodieLogFile; -import com.uber.hoodie.common.model.HoodieTableType; -import com.uber.hoodie.common.model.HoodieTestUtils; -import com.uber.hoodie.common.table.HoodieTableMetaClient; -import com.uber.hoodie.common.table.HoodieTimeline; -import com.uber.hoodie.common.table.SyncableFileSystemView; -import com.uber.hoodie.common.table.TableFileSystemView; -import com.uber.hoodie.common.table.timeline.HoodieActiveTimeline; -import com.uber.hoodie.common.table.timeline.HoodieInstant; -import com.uber.hoodie.common.table.timeline.HoodieInstant.State; -import com.uber.hoodie.common.util.AvroUtils; -import com.uber.hoodie.common.util.CompactionUtils; -import com.uber.hoodie.common.util.FSUtils; -import com.uber.hoodie.common.util.Option; -import com.uber.hoodie.common.util.collection.Pair; import java.io.File; import java.io.IOException; import java.util.ArrayList; @@ -58,6 +37,27 @@ import java.util.stream.Collectors; import java.util.stream.Stream; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.Path; +import org.apache.hudi.avro.model.HoodieCompactionPlan; +import org.apache.hudi.common.model.CompactionOperation; +import org.apache.hudi.common.model.FileSlice; +import org.apache.hudi.common.model.HoodieDataFile; +import org.apache.hudi.common.model.HoodieFileGroup; +import org.apache.hudi.common.model.HoodieFileGroupId; +import org.apache.hudi.common.model.HoodieLogFile; +import org.apache.hudi.common.model.HoodieTableType; +import org.apache.hudi.common.model.HoodieTestUtils; +import org.apache.hudi.common.table.HoodieTableMetaClient; +import org.apache.hudi.common.table.HoodieTimeline; +import org.apache.hudi.common.table.SyncableFileSystemView; +import org.apache.hudi.common.table.TableFileSystemView; +import org.apache.hudi.common.table.timeline.HoodieActiveTimeline; +import org.apache.hudi.common.table.timeline.HoodieInstant; +import org.apache.hudi.common.table.timeline.HoodieInstant.State; +import org.apache.hudi.common.util.AvroUtils; +import org.apache.hudi.common.util.CompactionUtils; +import org.apache.hudi.common.util.FSUtils; +import org.apache.hudi.common.util.Option; +import org.apache.hudi.common.util.collection.Pair; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; import org.junit.Assert; diff --git a/hoodie-common/src/test/java/com/uber/hoodie/common/table/view/IncrementalFSViewSyncTest.java b/hudi-common/src/test/java/org/apache/hudi/common/table/view/IncrementalFSViewSyncTest.java similarity index 95% rename from hoodie-common/src/test/java/com/uber/hoodie/common/table/view/IncrementalFSViewSyncTest.java rename to hudi-common/src/test/java/org/apache/hudi/common/table/view/IncrementalFSViewSyncTest.java index 349c6cca7..e90e2f113 100644 --- a/hoodie-common/src/test/java/com/uber/hoodie/common/table/view/IncrementalFSViewSyncTest.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/table/view/IncrementalFSViewSyncTest.java @@ -16,40 +16,13 @@ * limitations under the License. */ -package com.uber.hoodie.common.table.view; +package org.apache.hudi.common.table.view; -import static com.uber.hoodie.common.table.HoodieTimeline.COMPACTION_ACTION; +import static org.apache.hudi.common.table.HoodieTimeline.COMPACTION_ACTION; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Iterators; -import com.uber.hoodie.avro.model.HoodieCleanMetadata; -import com.uber.hoodie.avro.model.HoodieCompactionPlan; -import com.uber.hoodie.avro.model.HoodieRestoreMetadata; -import com.uber.hoodie.avro.model.HoodieRollbackMetadata; -import com.uber.hoodie.common.HoodieCleanStat; -import com.uber.hoodie.common.HoodieRollbackStat; -import com.uber.hoodie.common.model.CompactionOperation; -import com.uber.hoodie.common.model.FileSlice; -import com.uber.hoodie.common.model.HoodieCleaningPolicy; -import com.uber.hoodie.common.model.HoodieCommitMetadata; -import com.uber.hoodie.common.model.HoodieDataFile; -import com.uber.hoodie.common.model.HoodieFileGroup; -import com.uber.hoodie.common.model.HoodieFileGroupId; -import com.uber.hoodie.common.model.HoodieTableType; -import com.uber.hoodie.common.model.HoodieTestUtils; -import com.uber.hoodie.common.model.HoodieWriteStat; -import com.uber.hoodie.common.table.HoodieTableMetaClient; -import com.uber.hoodie.common.table.HoodieTimeline; -import com.uber.hoodie.common.table.SyncableFileSystemView; -import com.uber.hoodie.common.table.timeline.HoodieInstant; -import com.uber.hoodie.common.table.timeline.HoodieInstant.State; -import com.uber.hoodie.common.util.AvroUtils; -import com.uber.hoodie.common.util.CompactionUtils; -import com.uber.hoodie.common.util.FSUtils; -import com.uber.hoodie.common.util.Option; -import com.uber.hoodie.common.util.collection.Pair; -import com.uber.hoodie.exception.HoodieException; import java.io.File; import java.io.IOException; import java.nio.charset.StandardCharsets; @@ -63,6 +36,33 @@ import java.util.UUID; import java.util.stream.Collectors; import java.util.stream.IntStream; import org.apache.hadoop.fs.Path; +import org.apache.hudi.avro.model.HoodieCleanMetadata; +import org.apache.hudi.avro.model.HoodieCompactionPlan; +import org.apache.hudi.avro.model.HoodieRestoreMetadata; +import org.apache.hudi.avro.model.HoodieRollbackMetadata; +import org.apache.hudi.common.HoodieCleanStat; +import org.apache.hudi.common.HoodieRollbackStat; +import org.apache.hudi.common.model.CompactionOperation; +import org.apache.hudi.common.model.FileSlice; +import org.apache.hudi.common.model.HoodieCleaningPolicy; +import org.apache.hudi.common.model.HoodieCommitMetadata; +import org.apache.hudi.common.model.HoodieDataFile; +import org.apache.hudi.common.model.HoodieFileGroup; +import org.apache.hudi.common.model.HoodieFileGroupId; +import org.apache.hudi.common.model.HoodieTableType; +import org.apache.hudi.common.model.HoodieTestUtils; +import org.apache.hudi.common.model.HoodieWriteStat; +import org.apache.hudi.common.table.HoodieTableMetaClient; +import org.apache.hudi.common.table.HoodieTimeline; +import org.apache.hudi.common.table.SyncableFileSystemView; +import org.apache.hudi.common.table.timeline.HoodieInstant; +import org.apache.hudi.common.table.timeline.HoodieInstant.State; +import org.apache.hudi.common.util.AvroUtils; +import org.apache.hudi.common.util.CompactionUtils; +import org.apache.hudi.common.util.FSUtils; +import org.apache.hudi.common.util.Option; +import org.apache.hudi.common.util.collection.Pair; +import org.apache.hudi.exception.HoodieException; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; import org.junit.Assert; diff --git a/hoodie-common/src/test/java/com/uber/hoodie/common/table/view/RocksDBBasedIncrementalFSViewSyncTest.java b/hudi-common/src/test/java/org/apache/hudi/common/table/view/RocksDBBasedIncrementalFSViewSyncTest.java similarity index 85% rename from hoodie-common/src/test/java/com/uber/hoodie/common/table/view/RocksDBBasedIncrementalFSViewSyncTest.java rename to hudi-common/src/test/java/org/apache/hudi/common/table/view/RocksDBBasedIncrementalFSViewSyncTest.java index a7de02511..a78a3084e 100644 --- a/hoodie-common/src/test/java/com/uber/hoodie/common/table/view/RocksDBBasedIncrementalFSViewSyncTest.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/table/view/RocksDBBasedIncrementalFSViewSyncTest.java @@ -16,12 +16,12 @@ * limitations under the License. */ -package com.uber.hoodie.common.table.view; +package org.apache.hudi.common.table.view; -import com.uber.hoodie.common.table.HoodieTableMetaClient; -import com.uber.hoodie.common.table.HoodieTimeline; -import com.uber.hoodie.common.table.SyncableFileSystemView; import java.io.IOException; +import org.apache.hudi.common.table.HoodieTableMetaClient; +import org.apache.hudi.common.table.HoodieTimeline; +import org.apache.hudi.common.table.SyncableFileSystemView; public class RocksDBBasedIncrementalFSViewSyncTest extends IncrementalFSViewSyncTest { diff --git a/hoodie-common/src/test/java/com/uber/hoodie/common/table/view/RocksDbBasedFileSystemViewTest.java b/hudi-common/src/test/java/org/apache/hudi/common/table/view/RocksDbBasedFileSystemViewTest.java similarity index 88% rename from hoodie-common/src/test/java/com/uber/hoodie/common/table/view/RocksDbBasedFileSystemViewTest.java rename to hudi-common/src/test/java/org/apache/hudi/common/table/view/RocksDbBasedFileSystemViewTest.java index 3bee35983..479a5cb49 100644 --- a/hoodie-common/src/test/java/com/uber/hoodie/common/table/view/RocksDbBasedFileSystemViewTest.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/table/view/RocksDbBasedFileSystemViewTest.java @@ -16,11 +16,11 @@ * limitations under the License. */ -package com.uber.hoodie.common.table.view; +package org.apache.hudi.common.table.view; -import com.uber.hoodie.common.table.HoodieTimeline; -import com.uber.hoodie.common.table.SyncableFileSystemView; import java.io.IOException; +import org.apache.hudi.common.table.HoodieTimeline; +import org.apache.hudi.common.table.SyncableFileSystemView; public class RocksDbBasedFileSystemViewTest extends HoodieTableFileSystemViewTest { diff --git a/hoodie-common/src/test/java/com/uber/hoodie/common/table/view/SpillableMapBasedFileSystemViewTest.java b/hudi-common/src/test/java/org/apache/hudi/common/table/view/SpillableMapBasedFileSystemViewTest.java similarity index 88% rename from hoodie-common/src/test/java/com/uber/hoodie/common/table/view/SpillableMapBasedFileSystemViewTest.java rename to hudi-common/src/test/java/org/apache/hudi/common/table/view/SpillableMapBasedFileSystemViewTest.java index e6d13bcb6..db2daf6f5 100644 --- a/hoodie-common/src/test/java/com/uber/hoodie/common/table/view/SpillableMapBasedFileSystemViewTest.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/table/view/SpillableMapBasedFileSystemViewTest.java @@ -16,10 +16,10 @@ * limitations under the License. */ -package com.uber.hoodie.common.table.view; +package org.apache.hudi.common.table.view; -import com.uber.hoodie.common.table.HoodieTimeline; -import com.uber.hoodie.common.table.SyncableFileSystemView; +import org.apache.hudi.common.table.HoodieTimeline; +import org.apache.hudi.common.table.SyncableFileSystemView; public class SpillableMapBasedFileSystemViewTest extends HoodieTableFileSystemViewTest { diff --git a/hoodie-common/src/test/java/com/uber/hoodie/common/table/view/SpillableMapBasedIncrementalFSViewSyncTest.java b/hudi-common/src/test/java/org/apache/hudi/common/table/view/SpillableMapBasedIncrementalFSViewSyncTest.java similarity index 85% rename from hoodie-common/src/test/java/com/uber/hoodie/common/table/view/SpillableMapBasedIncrementalFSViewSyncTest.java rename to hudi-common/src/test/java/org/apache/hudi/common/table/view/SpillableMapBasedIncrementalFSViewSyncTest.java index bcfe1f3f3..ce3cbdd39 100644 --- a/hoodie-common/src/test/java/com/uber/hoodie/common/table/view/SpillableMapBasedIncrementalFSViewSyncTest.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/table/view/SpillableMapBasedIncrementalFSViewSyncTest.java @@ -16,11 +16,11 @@ * limitations under the License. */ -package com.uber.hoodie.common.table.view; +package org.apache.hudi.common.table.view; -import com.uber.hoodie.common.table.HoodieTableMetaClient; -import com.uber.hoodie.common.table.HoodieTimeline; -import com.uber.hoodie.common.table.SyncableFileSystemView; +import org.apache.hudi.common.table.HoodieTableMetaClient; +import org.apache.hudi.common.table.HoodieTimeline; +import org.apache.hudi.common.table.SyncableFileSystemView; public class SpillableMapBasedIncrementalFSViewSyncTest extends IncrementalFSViewSyncTest { diff --git a/hoodie-common/src/test/java/com/uber/hoodie/common/util/CompactionTestUtils.java b/hudi-common/src/test/java/org/apache/hudi/common/util/CompactionTestUtils.java similarity index 90% rename from hoodie-common/src/test/java/com/uber/hoodie/common/util/CompactionTestUtils.java rename to hudi-common/src/test/java/org/apache/hudi/common/util/CompactionTestUtils.java index 1cd5d9c9f..d320a7697 100644 --- a/hoodie-common/src/test/java/com/uber/hoodie/common/util/CompactionTestUtils.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/util/CompactionTestUtils.java @@ -16,26 +16,14 @@ * limitations under the License. */ -package com.uber.hoodie.common.util; +package org.apache.hudi.common.util; -import static com.uber.hoodie.common.model.HoodieTestUtils.DEFAULT_PARTITION_PATHS; -import static com.uber.hoodie.common.table.HoodieTimeline.COMPACTION_ACTION; -import static com.uber.hoodie.common.table.HoodieTimeline.DELTA_COMMIT_ACTION; +import static org.apache.hudi.common.model.HoodieTestUtils.DEFAULT_PARTITION_PATHS; +import static org.apache.hudi.common.table.HoodieTimeline.COMPACTION_ACTION; +import static org.apache.hudi.common.table.HoodieTimeline.DELTA_COMMIT_ACTION; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; -import com.uber.hoodie.avro.model.HoodieCompactionOperation; -import com.uber.hoodie.avro.model.HoodieCompactionPlan; -import com.uber.hoodie.common.model.FileSlice; -import com.uber.hoodie.common.model.HoodieDataFile; -import com.uber.hoodie.common.model.HoodieFileGroupId; -import com.uber.hoodie.common.model.HoodieLogFile; -import com.uber.hoodie.common.model.HoodieTestUtils; -import com.uber.hoodie.common.table.HoodieTableMetaClient; -import com.uber.hoodie.common.table.timeline.HoodieInstant; -import com.uber.hoodie.common.table.timeline.HoodieInstant.State; -import com.uber.hoodie.common.util.collection.Pair; -import com.uber.hoodie.exception.HoodieIOException; import java.io.IOException; import java.util.Arrays; import java.util.HashMap; @@ -46,6 +34,18 @@ import java.util.stream.Collectors; import java.util.stream.IntStream; import java.util.stream.Stream; import org.apache.hadoop.fs.Path; +import org.apache.hudi.avro.model.HoodieCompactionOperation; +import org.apache.hudi.avro.model.HoodieCompactionPlan; +import org.apache.hudi.common.model.FileSlice; +import org.apache.hudi.common.model.HoodieDataFile; +import org.apache.hudi.common.model.HoodieFileGroupId; +import org.apache.hudi.common.model.HoodieLogFile; +import org.apache.hudi.common.model.HoodieTestUtils; +import org.apache.hudi.common.table.HoodieTableMetaClient; +import org.apache.hudi.common.table.timeline.HoodieInstant; +import org.apache.hudi.common.table.timeline.HoodieInstant.State; +import org.apache.hudi.common.util.collection.Pair; +import org.apache.hudi.exception.HoodieIOException; import org.junit.Assert; public class CompactionTestUtils { diff --git a/hoodie-common/src/test/java/com/uber/hoodie/common/util/SchemaTestUtil.java b/hudi-common/src/test/java/org/apache/hudi/common/util/SchemaTestUtil.java similarity index 95% rename from hoodie-common/src/test/java/com/uber/hoodie/common/util/SchemaTestUtil.java rename to hudi-common/src/test/java/org/apache/hudi/common/util/SchemaTestUtil.java index ed5491ee8..4b62f8152 100644 --- a/hoodie-common/src/test/java/com/uber/hoodie/common/util/SchemaTestUtil.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/util/SchemaTestUtil.java @@ -16,14 +16,8 @@ * limitations under the License. */ -package com.uber.hoodie.common.util; +package org.apache.hudi.common.util; -import com.uber.hoodie.avro.MercifulJsonConverter; -import com.uber.hoodie.common.model.HoodieAvroPayload; -import com.uber.hoodie.common.model.HoodieKey; -import com.uber.hoodie.common.model.HoodieRecord; -import com.uber.hoodie.common.table.timeline.HoodieActiveTimeline; -import com.uber.hoodie.exception.HoodieIOException; import java.io.IOException; import java.net.URI; import java.net.URISyntaxException; @@ -44,6 +38,12 @@ import org.apache.avro.generic.GenericDatumReader; import org.apache.avro.generic.GenericRecord; import org.apache.avro.generic.IndexedRecord; import org.apache.avro.io.DecoderFactory; +import org.apache.hudi.avro.MercifulJsonConverter; +import org.apache.hudi.common.model.HoodieAvroPayload; +import org.apache.hudi.common.model.HoodieKey; +import org.apache.hudi.common.model.HoodieRecord; +import org.apache.hudi.common.table.timeline.HoodieActiveTimeline; +import org.apache.hudi.exception.HoodieIOException; public class SchemaTestUtil { diff --git a/hoodie-common/src/test/java/com/uber/hoodie/common/util/SpillableMapTestUtils.java b/hudi-common/src/test/java/org/apache/hudi/common/util/SpillableMapTestUtils.java similarity index 86% rename from hoodie-common/src/test/java/com/uber/hoodie/common/util/SpillableMapTestUtils.java rename to hudi-common/src/test/java/org/apache/hudi/common/util/SpillableMapTestUtils.java index 1b7ea41b9..62a36e8e4 100644 --- a/hoodie-common/src/test/java/com/uber/hoodie/common/util/SpillableMapTestUtils.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/util/SpillableMapTestUtils.java @@ -16,18 +16,18 @@ * limitations under the License. */ -package com.uber.hoodie.common.util; +package org.apache.hudi.common.util; -import com.uber.hoodie.common.model.HoodieAvroPayload; -import com.uber.hoodie.common.model.HoodieKey; -import com.uber.hoodie.common.model.HoodieRecord; -import com.uber.hoodie.common.model.HoodieRecordLocation; -import com.uber.hoodie.common.model.HoodieRecordPayload; import java.util.ArrayList; import java.util.List; import java.util.Map; import org.apache.avro.generic.GenericRecord; import org.apache.avro.generic.IndexedRecord; +import org.apache.hudi.common.model.HoodieAvroPayload; +import org.apache.hudi.common.model.HoodieKey; +import org.apache.hudi.common.model.HoodieRecord; +import org.apache.hudi.common.model.HoodieRecordLocation; +import org.apache.hudi.common.model.HoodieRecordPayload; public class SpillableMapTestUtils { diff --git a/hoodie-common/src/test/java/com/uber/hoodie/common/util/TestCompactionUtils.java b/hudi-common/src/test/java/org/apache/hudi/common/util/TestCompactionUtils.java similarity index 92% rename from hoodie-common/src/test/java/com/uber/hoodie/common/util/TestCompactionUtils.java rename to hudi-common/src/test/java/org/apache/hudi/common/util/TestCompactionUtils.java index 4f369375a..4c37d8a45 100644 --- a/hoodie-common/src/test/java/com/uber/hoodie/common/util/TestCompactionUtils.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/util/TestCompactionUtils.java @@ -16,25 +16,15 @@ * limitations under the License. */ -package com.uber.hoodie.common.util; +package org.apache.hudi.common.util; -import static com.uber.hoodie.common.model.HoodieTestUtils.DEFAULT_PARTITION_PATHS; -import static com.uber.hoodie.common.model.HoodieTestUtils.getDefaultHadoopConf; -import static com.uber.hoodie.common.util.CompactionTestUtils.createCompactionPlan; -import static com.uber.hoodie.common.util.CompactionTestUtils.scheduleCompaction; -import static com.uber.hoodie.common.util.CompactionTestUtils.setupAndValidateCompactionOperations; +import static org.apache.hudi.common.model.HoodieTestUtils.DEFAULT_PARTITION_PATHS; +import static org.apache.hudi.common.model.HoodieTestUtils.getDefaultHadoopConf; +import static org.apache.hudi.common.util.CompactionTestUtils.createCompactionPlan; +import static org.apache.hudi.common.util.CompactionTestUtils.scheduleCompaction; +import static org.apache.hudi.common.util.CompactionTestUtils.setupAndValidateCompactionOperations; import com.google.common.collect.ImmutableMap; -import com.uber.hoodie.avro.model.HoodieCompactionOperation; -import com.uber.hoodie.avro.model.HoodieCompactionPlan; -import com.uber.hoodie.common.model.FileSlice; -import com.uber.hoodie.common.model.HoodieFileGroupId; -import com.uber.hoodie.common.model.HoodieLogFile; -import com.uber.hoodie.common.model.HoodieTableType; -import com.uber.hoodie.common.model.HoodieTestUtils; -import com.uber.hoodie.common.table.HoodieTableMetaClient; -import com.uber.hoodie.common.util.CompactionTestUtils.TestHoodieDataFile; -import com.uber.hoodie.common.util.collection.Pair; import java.io.IOException; import java.util.Arrays; import java.util.List; @@ -43,6 +33,16 @@ import java.util.function.Function; import java.util.stream.Collectors; import java.util.stream.IntStream; import org.apache.hadoop.fs.Path; +import org.apache.hudi.avro.model.HoodieCompactionOperation; +import org.apache.hudi.avro.model.HoodieCompactionPlan; +import org.apache.hudi.common.model.FileSlice; +import org.apache.hudi.common.model.HoodieFileGroupId; +import org.apache.hudi.common.model.HoodieLogFile; +import org.apache.hudi.common.model.HoodieTableType; +import org.apache.hudi.common.model.HoodieTestUtils; +import org.apache.hudi.common.table.HoodieTableMetaClient; +import org.apache.hudi.common.util.CompactionTestUtils.TestHoodieDataFile; +import org.apache.hudi.common.util.collection.Pair; import org.junit.Assert; import org.junit.Before; import org.junit.Rule; diff --git a/hoodie-common/src/test/java/com/uber/hoodie/common/util/TestDFSPropertiesConfiguration.java b/hudi-common/src/test/java/org/apache/hudi/common/util/TestDFSPropertiesConfiguration.java similarity index 98% rename from hoodie-common/src/test/java/com/uber/hoodie/common/util/TestDFSPropertiesConfiguration.java rename to hudi-common/src/test/java/org/apache/hudi/common/util/TestDFSPropertiesConfiguration.java index 8ecc0a1bf..36d37a686 100644 --- a/hoodie-common/src/test/java/com/uber/hoodie/common/util/TestDFSPropertiesConfiguration.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/util/TestDFSPropertiesConfiguration.java @@ -16,17 +16,17 @@ * limitations under the License. */ -package com.uber.hoodie.common.util; +package org.apache.hudi.common.util; import static org.junit.Assert.assertEquals; import static org.junit.Assert.fail; -import com.uber.hoodie.common.minicluster.HdfsTestService; import java.io.IOException; import java.io.PrintStream; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.hudi.common.minicluster.HdfsTestService; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Test; diff --git a/hoodie-common/src/test/java/com/uber/hoodie/common/util/TestFSUtils.java b/hudi-common/src/test/java/org/apache/hudi/common/util/TestFSUtils.java similarity index 92% rename from hoodie-common/src/test/java/com/uber/hoodie/common/util/TestFSUtils.java rename to hudi-common/src/test/java/org/apache/hudi/common/util/TestFSUtils.java index 69ada2543..5a315e612 100644 --- a/hoodie-common/src/test/java/com/uber/hoodie/common/util/TestFSUtils.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/util/TestFSUtils.java @@ -16,15 +16,11 @@ * limitations under the License. */ -package com.uber.hoodie.common.util; +package org.apache.hudi.common.util; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; -import com.uber.hoodie.common.model.HoodieLogFile; -import com.uber.hoodie.common.model.HoodieTestUtils; -import com.uber.hoodie.common.table.HoodieTableMetaClient; -import com.uber.hoodie.exception.HoodieException; import java.io.IOException; import java.text.SimpleDateFormat; import java.util.ArrayList; @@ -36,6 +32,10 @@ import java.util.regex.Pattern; import java.util.stream.Collectors; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; +import org.apache.hudi.common.model.HoodieLogFile; +import org.apache.hudi.common.model.HoodieTestUtils; +import org.apache.hudi.common.table.HoodieTableMetaClient; +import org.apache.hudi.exception.HoodieException; import org.junit.Assert; import org.junit.Rule; import org.junit.Test; @@ -196,9 +196,9 @@ public class TestFSUtils { String oldLogFile = makeOldLogFileName(fileName, ".log", "100", 1); Path rlPath = new Path(new Path(partitionPath), oldLogFile); Assert.assertTrue(FSUtils.isLogFile(rlPath)); - Assert.assertEquals(fileName, FSUtils.getFileIdFromLogPath(rlPath)); - Assert.assertEquals("100", FSUtils.getBaseCommitTimeFromLogPath(rlPath)); - Assert.assertEquals(1, FSUtils.getFileVersionFromLog(rlPath)); + assertEquals(fileName, FSUtils.getFileIdFromLogPath(rlPath)); + assertEquals("100", FSUtils.getBaseCommitTimeFromLogPath(rlPath)); + assertEquals(1, FSUtils.getFileVersionFromLog(rlPath)); Assert.assertNull(FSUtils.getTaskPartitionIdFromLogPath(rlPath)); Assert.assertNull(FSUtils.getStageIdFromLogPath(rlPath)); Assert.assertNull(FSUtils.getTaskAttemptIdFromLogPath(rlPath)); @@ -214,12 +214,12 @@ public class TestFSUtils { System.out.println("Log File =" + logFile); Path rlPath = new Path(new Path(partitionPath), logFile); Assert.assertTrue(FSUtils.isLogFile(rlPath)); - Assert.assertEquals(fileName, FSUtils.getFileIdFromLogPath(rlPath)); - Assert.assertEquals("100", FSUtils.getBaseCommitTimeFromLogPath(rlPath)); - Assert.assertEquals(2, FSUtils.getFileVersionFromLog(rlPath)); - Assert.assertEquals(new Integer(1), FSUtils.getTaskPartitionIdFromLogPath(rlPath)); - Assert.assertEquals(new Integer(0), FSUtils.getStageIdFromLogPath(rlPath)); - Assert.assertEquals(new Integer(1), FSUtils.getTaskAttemptIdFromLogPath(rlPath)); + assertEquals(fileName, FSUtils.getFileIdFromLogPath(rlPath)); + assertEquals("100", FSUtils.getBaseCommitTimeFromLogPath(rlPath)); + assertEquals(2, FSUtils.getFileVersionFromLog(rlPath)); + assertEquals(new Integer(1), FSUtils.getTaskPartitionIdFromLogPath(rlPath)); + assertEquals(new Integer(0), FSUtils.getStageIdFromLogPath(rlPath)); + assertEquals(new Integer(1), FSUtils.getTaskAttemptIdFromLogPath(rlPath)); } /** diff --git a/hoodie-common/src/test/java/com/uber/hoodie/common/util/TestHoodieAvroUtils.java b/hudi-common/src/test/java/org/apache/hudi/common/util/TestHoodieAvroUtils.java similarity index 98% rename from hoodie-common/src/test/java/com/uber/hoodie/common/util/TestHoodieAvroUtils.java rename to hudi-common/src/test/java/org/apache/hudi/common/util/TestHoodieAvroUtils.java index 9e95683db..002100a67 100644 --- a/hoodie-common/src/test/java/com/uber/hoodie/common/util/TestHoodieAvroUtils.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/util/TestHoodieAvroUtils.java @@ -16,10 +16,9 @@ * limitations under the License. */ -package com.uber.hoodie.common.util; +package org.apache.hudi.common.util; import java.util.Map; - import org.apache.avro.Schema; import org.codehaus.jackson.JsonNode; import org.junit.Assert; diff --git a/hoodie-common/src/test/java/com/uber/hoodie/common/util/TestNumericUtils.java b/hudi-common/src/test/java/org/apache/hudi/common/util/TestNumericUtils.java similarity index 97% rename from hoodie-common/src/test/java/com/uber/hoodie/common/util/TestNumericUtils.java rename to hudi-common/src/test/java/org/apache/hudi/common/util/TestNumericUtils.java index 1b79a1f48..ad4c8eac7 100644 --- a/hoodie-common/src/test/java/com/uber/hoodie/common/util/TestNumericUtils.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/util/TestNumericUtils.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package com.uber.hoodie.common.util; +package org.apache.hudi.common.util; import static org.junit.Assert.assertTrue; diff --git a/hoodie-common/src/test/java/com/uber/hoodie/common/util/TestParquetUtils.java b/hudi-common/src/test/java/org/apache/hudi/common/util/TestParquetUtils.java similarity index 94% rename from hoodie-common/src/test/java/com/uber/hoodie/common/util/TestParquetUtils.java rename to hudi-common/src/test/java/org/apache/hudi/common/util/TestParquetUtils.java index d0abed9d9..a63f91292 100644 --- a/hoodie-common/src/test/java/com/uber/hoodie/common/util/TestParquetUtils.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/util/TestParquetUtils.java @@ -16,15 +16,11 @@ * limitations under the License. */ -package com.uber.hoodie.common.util; +package org.apache.hudi.common.util; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; -import com.uber.hoodie.avro.HoodieAvroWriteSupport; -import com.uber.hoodie.common.BloomFilter; -import com.uber.hoodie.common.model.HoodieRecord; -import com.uber.hoodie.common.model.HoodieTestUtils; import java.io.IOException; import java.util.ArrayList; import java.util.Collections; @@ -36,6 +32,10 @@ import org.apache.avro.Schema; import org.apache.avro.generic.GenericData; import org.apache.avro.generic.GenericRecord; import org.apache.hadoop.fs.Path; +import org.apache.hudi.avro.HoodieAvroWriteSupport; +import org.apache.hudi.common.BloomFilter; +import org.apache.hudi.common.model.HoodieRecord; +import org.apache.hudi.common.model.HoodieTestUtils; import org.apache.parquet.avro.AvroSchemaConverter; import org.apache.parquet.hadoop.ParquetWriter; import org.apache.parquet.hadoop.metadata.CompressionCodecName; diff --git a/hoodie-common/src/test/java/com/uber/hoodie/common/util/TestRecord.java b/hudi-common/src/test/java/org/apache/hudi/common/util/TestRecord.java similarity index 99% rename from hoodie-common/src/test/java/com/uber/hoodie/common/util/TestRecord.java rename to hudi-common/src/test/java/org/apache/hudi/common/util/TestRecord.java index dc2a461e9..b0c4a14f2 100644 --- a/hoodie-common/src/test/java/com/uber/hoodie/common/util/TestRecord.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/util/TestRecord.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package com.uber.hoodie.common.util; +package org.apache.hudi.common.util; import com.fasterxml.jackson.annotation.JsonAutoDetect; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; diff --git a/hoodie-common/src/test/java/com/uber/hoodie/common/util/TestRocksDBManager.java b/hudi-common/src/test/java/org/apache/hudi/common/util/TestRocksDBManager.java similarity index 97% rename from hoodie-common/src/test/java/com/uber/hoodie/common/util/TestRocksDBManager.java rename to hudi-common/src/test/java/org/apache/hudi/common/util/TestRocksDBManager.java index dd239e184..83519a300 100644 --- a/hoodie-common/src/test/java/com/uber/hoodie/common/util/TestRocksDBManager.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/util/TestRocksDBManager.java @@ -16,10 +16,8 @@ * limitations under the License. */ -package com.uber.hoodie.common.util; +package org.apache.hudi.common.util; -import com.uber.hoodie.common.table.view.FileSystemViewStorageConfig; -import com.uber.hoodie.common.util.collection.Pair; import java.io.File; import java.io.IOException; import java.io.Serializable; @@ -31,6 +29,8 @@ import java.util.Objects; import java.util.UUID; import java.util.stream.Collectors; import java.util.stream.IntStream; +import org.apache.hudi.common.table.view.FileSystemViewStorageConfig; +import org.apache.hudi.common.util.collection.Pair; import org.junit.AfterClass; import org.junit.Assert; import org.junit.Test; diff --git a/hoodie-common/src/test/java/com/uber/hoodie/common/util/TestSerializationUtils.java b/hudi-common/src/test/java/org/apache/hudi/common/util/TestSerializationUtils.java similarity index 98% rename from hoodie-common/src/test/java/com/uber/hoodie/common/util/TestSerializationUtils.java rename to hudi-common/src/test/java/org/apache/hudi/common/util/TestSerializationUtils.java index 858d5ee42..d97535def 100644 --- a/hoodie-common/src/test/java/com/uber/hoodie/common/util/TestSerializationUtils.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/util/TestSerializationUtils.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package com.uber.hoodie.common.util; +package org.apache.hudi.common.util; import java.io.IOException; import java.util.Arrays; diff --git a/hoodie-common/src/test/java/com/uber/hoodie/common/util/collection/TestDiskBasedMap.java b/hudi-common/src/test/java/org/apache/hudi/common/util/collection/TestDiskBasedMap.java similarity index 91% rename from hoodie-common/src/test/java/com/uber/hoodie/common/util/collection/TestDiskBasedMap.java rename to hudi-common/src/test/java/org/apache/hudi/common/util/collection/TestDiskBasedMap.java index e55eb94b8..ab49307b2 100644 --- a/hoodie-common/src/test/java/com/uber/hoodie/common/util/collection/TestDiskBasedMap.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/util/collection/TestDiskBasedMap.java @@ -16,24 +16,12 @@ * limitations under the License. */ -package com.uber.hoodie.common.util.collection; +package org.apache.hudi.common.util.collection; -import static com.uber.hoodie.common.util.SchemaTestUtil.getSimpleSchema; +import static org.apache.hudi.common.util.SchemaTestUtil.getSimpleSchema; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; -import com.uber.hoodie.common.model.AvroBinaryTestPayload; -import com.uber.hoodie.common.model.HoodieAvroPayload; -import com.uber.hoodie.common.model.HoodieKey; -import com.uber.hoodie.common.model.HoodieRecord; -import com.uber.hoodie.common.model.HoodieRecordPayload; -import com.uber.hoodie.common.table.timeline.HoodieActiveTimeline; -import com.uber.hoodie.common.util.HoodieAvroUtils; -import com.uber.hoodie.common.util.HoodieRecordSizeEstimator; -import com.uber.hoodie.common.util.Option; -import com.uber.hoodie.common.util.SchemaTestUtil; -import com.uber.hoodie.common.util.SpillableMapTestUtils; -import com.uber.hoodie.common.util.SpillableMapUtils; import java.io.IOException; import java.io.UncheckedIOException; import java.net.URISyntaxException; @@ -47,6 +35,18 @@ import java.util.stream.Collectors; import org.apache.avro.Schema; import org.apache.avro.generic.GenericRecord; import org.apache.avro.generic.IndexedRecord; +import org.apache.hudi.common.model.AvroBinaryTestPayload; +import org.apache.hudi.common.model.HoodieAvroPayload; +import org.apache.hudi.common.model.HoodieKey; +import org.apache.hudi.common.model.HoodieRecord; +import org.apache.hudi.common.model.HoodieRecordPayload; +import org.apache.hudi.common.table.timeline.HoodieActiveTimeline; +import org.apache.hudi.common.util.HoodieAvroUtils; +import org.apache.hudi.common.util.HoodieRecordSizeEstimator; +import org.apache.hudi.common.util.Option; +import org.apache.hudi.common.util.SchemaTestUtil; +import org.apache.hudi.common.util.SpillableMapTestUtils; +import org.apache.hudi.common.util.SpillableMapUtils; import org.junit.Ignore; import org.junit.Test; diff --git a/hoodie-common/src/test/java/com/uber/hoodie/common/util/collection/TestExternalSpillableMap.java b/hudi-common/src/test/java/org/apache/hudi/common/util/collection/TestExternalSpillableMap.java similarity index 95% rename from hoodie-common/src/test/java/com/uber/hoodie/common/util/collection/TestExternalSpillableMap.java rename to hudi-common/src/test/java/org/apache/hudi/common/util/collection/TestExternalSpillableMap.java index 05c2a0745..6b61b9ee4 100644 --- a/hoodie-common/src/test/java/com/uber/hoodie/common/util/collection/TestExternalSpillableMap.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/util/collection/TestExternalSpillableMap.java @@ -16,23 +16,12 @@ * limitations under the License. */ -package com.uber.hoodie.common.util.collection; +package org.apache.hudi.common.util.collection; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; -import com.uber.hoodie.common.model.HoodieAvroPayload; -import com.uber.hoodie.common.model.HoodieKey; -import com.uber.hoodie.common.model.HoodieRecord; -import com.uber.hoodie.common.model.HoodieRecordPayload; -import com.uber.hoodie.common.table.timeline.HoodieActiveTimeline; -import com.uber.hoodie.common.util.DefaultSizeEstimator; -import com.uber.hoodie.common.util.HoodieAvroUtils; -import com.uber.hoodie.common.util.HoodieRecordSizeEstimator; -import com.uber.hoodie.common.util.Option; -import com.uber.hoodie.common.util.SchemaTestUtil; -import com.uber.hoodie.common.util.SpillableMapTestUtils; import java.io.File; import java.io.IOException; import java.io.UncheckedIOException; @@ -43,6 +32,17 @@ import java.util.List; import org.apache.avro.Schema; import org.apache.avro.generic.GenericRecord; import org.apache.avro.generic.IndexedRecord; +import org.apache.hudi.common.model.HoodieAvroPayload; +import org.apache.hudi.common.model.HoodieKey; +import org.apache.hudi.common.model.HoodieRecord; +import org.apache.hudi.common.model.HoodieRecordPayload; +import org.apache.hudi.common.table.timeline.HoodieActiveTimeline; +import org.apache.hudi.common.util.DefaultSizeEstimator; +import org.apache.hudi.common.util.HoodieAvroUtils; +import org.apache.hudi.common.util.HoodieRecordSizeEstimator; +import org.apache.hudi.common.util.Option; +import org.apache.hudi.common.util.SchemaTestUtil; +import org.apache.hudi.common.util.SpillableMapTestUtils; import org.junit.BeforeClass; import org.junit.FixMethodOrder; import org.junit.Test; diff --git a/hoodie-common/src/test/java/com/uber/hoodie/common/util/collection/TestRocksDbBasedMap.java b/hudi-common/src/test/java/org/apache/hudi/common/util/collection/TestRocksDbBasedMap.java similarity index 88% rename from hoodie-common/src/test/java/com/uber/hoodie/common/util/collection/TestRocksDbBasedMap.java rename to hudi-common/src/test/java/org/apache/hudi/common/util/collection/TestRocksDbBasedMap.java index e91625da4..5216a9ee1 100644 --- a/hoodie-common/src/test/java/com/uber/hoodie/common/util/collection/TestRocksDbBasedMap.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/util/collection/TestRocksDbBasedMap.java @@ -16,12 +16,8 @@ * limitations under the License. */ -package com.uber.hoodie.common.util.collection; +package org.apache.hudi.common.util.collection; -import com.uber.hoodie.common.model.HoodieRecord; -import com.uber.hoodie.common.model.HoodieRecordPayload; -import com.uber.hoodie.common.util.SchemaTestUtil; -import com.uber.hoodie.common.util.SpillableMapTestUtils; import java.io.IOException; import java.net.URISyntaxException; import java.util.ArrayList; @@ -29,6 +25,10 @@ import java.util.Iterator; import java.util.List; import org.apache.avro.generic.GenericRecord; import org.apache.avro.generic.IndexedRecord; +import org.apache.hudi.common.model.HoodieRecord; +import org.apache.hudi.common.model.HoodieRecordPayload; +import org.apache.hudi.common.util.SchemaTestUtil; +import org.apache.hudi.common.util.SpillableMapTestUtils; import org.junit.Assert; import org.junit.Test; diff --git a/hoodie-common/src/test/resources/complex-test-evolved.avsc b/hudi-common/src/test/resources/complex-test-evolved.avsc similarity index 100% rename from hoodie-common/src/test/resources/complex-test-evolved.avsc rename to hudi-common/src/test/resources/complex-test-evolved.avsc diff --git a/hoodie-common/src/test/resources/log4j-surefire-quiet.properties b/hudi-common/src/test/resources/log4j-surefire-quiet.properties similarity index 100% rename from hoodie-common/src/test/resources/log4j-surefire-quiet.properties rename to hudi-common/src/test/resources/log4j-surefire-quiet.properties diff --git a/hoodie-common/src/test/resources/log4j-surefire.properties b/hudi-common/src/test/resources/log4j-surefire.properties similarity index 100% rename from hoodie-common/src/test/resources/log4j-surefire.properties rename to hudi-common/src/test/resources/log4j-surefire.properties diff --git a/hoodie-common/src/test/resources/sample.data b/hudi-common/src/test/resources/sample.data similarity index 100% rename from hoodie-common/src/test/resources/sample.data rename to hudi-common/src/test/resources/sample.data diff --git a/hoodie-common/src/test/resources/simple-test-evolved.avsc b/hudi-common/src/test/resources/simple-test-evolved.avsc similarity index 100% rename from hoodie-common/src/test/resources/simple-test-evolved.avsc rename to hudi-common/src/test/resources/simple-test-evolved.avsc diff --git a/hoodie-common/src/test/resources/simple-test.avsc b/hudi-common/src/test/resources/simple-test.avsc similarity index 100% rename from hoodie-common/src/test/resources/simple-test.avsc rename to hudi-common/src/test/resources/simple-test.avsc diff --git a/hoodie-hadoop-mr/pom.xml b/hudi-hadoop-mr/pom.xml similarity index 92% rename from hoodie-hadoop-mr/pom.xml rename to hudi-hadoop-mr/pom.xml index b467fe51a..cc6cfa2c7 100644 --- a/hoodie-hadoop-mr/pom.xml +++ b/hudi-hadoop-mr/pom.xml @@ -17,13 +17,13 @@ --> - hoodie - com.uber.hoodie - 0.4.8-SNAPSHOT + hudi + org.apache.hudi + 0.5.0-SNAPSHOT 4.0.0 - hoodie-hadoop-mr + hudi-hadoop-mr ${project.basedir}/src/main/resources/META-INF @@ -31,13 +31,13 @@ - com.uber.hoodie - hoodie-common + org.apache.hudi + hudi-common ${project.version} - com.uber.hoodie - hoodie-common + org.apache.hudi + hudi-common ${project.version} tests test-jar diff --git a/hudi-hadoop-mr/src/main/java/com/uber/hoodie/hadoop/HoodieInputFormat.java b/hudi-hadoop-mr/src/main/java/com/uber/hoodie/hadoop/HoodieInputFormat.java new file mode 100644 index 000000000..1b1ad423b --- /dev/null +++ b/hudi-hadoop-mr/src/main/java/com/uber/hoodie/hadoop/HoodieInputFormat.java @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.uber.hoodie.hadoop; + +/** + * Temporary class to allow seamless migration of com.uber.hoodie to org.apache.hudi + */ +public class HoodieInputFormat extends org.apache.hudi.hadoop.HoodieInputFormat { + +} diff --git a/hudi-hadoop-mr/src/main/java/com/uber/hoodie/hadoop/realtime/HoodieRealtimeInputFormat.java b/hudi-hadoop-mr/src/main/java/com/uber/hoodie/hadoop/realtime/HoodieRealtimeInputFormat.java new file mode 100644 index 000000000..627876741 --- /dev/null +++ b/hudi-hadoop-mr/src/main/java/com/uber/hoodie/hadoop/realtime/HoodieRealtimeInputFormat.java @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.uber.hoodie.hadoop.realtime; + +/** + * Temporary class to allow seamless migration of com.uber.hoodie to org.apache.hudi + */ +public class HoodieRealtimeInputFormat extends org.apache.hudi.hadoop.realtime.HoodieRealtimeInputFormat { + +} diff --git a/hoodie-hadoop-mr/src/main/java/com/uber/hoodie/hadoop/HoodieHiveUtil.java b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/HoodieHiveUtil.java similarity index 98% rename from hoodie-hadoop-mr/src/main/java/com/uber/hoodie/hadoop/HoodieHiveUtil.java rename to hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/HoodieHiveUtil.java index b19fb2707..1db8c541d 100644 --- a/hoodie-hadoop-mr/src/main/java/com/uber/hoodie/hadoop/HoodieHiveUtil.java +++ b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/HoodieHiveUtil.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package com.uber.hoodie.hadoop; +package org.apache.hudi.hadoop; import org.apache.hadoop.fs.Path; import org.apache.hadoop.mapreduce.JobContext; diff --git a/hoodie-hadoop-mr/src/main/java/com/uber/hoodie/hadoop/HoodieInputFormat.java b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/HoodieInputFormat.java similarity index 93% rename from hoodie-hadoop-mr/src/main/java/com/uber/hoodie/hadoop/HoodieInputFormat.java rename to hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/HoodieInputFormat.java index c15663041..9f8e4d550 100644 --- a/hoodie-hadoop-mr/src/main/java/com/uber/hoodie/hadoop/HoodieInputFormat.java +++ b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/HoodieInputFormat.java @@ -16,18 +16,8 @@ * limitations under the License. */ -package com.uber.hoodie.hadoop; +package org.apache.hudi.hadoop; -import com.uber.hoodie.common.model.HoodieDataFile; -import com.uber.hoodie.common.model.HoodiePartitionMetadata; -import com.uber.hoodie.common.table.HoodieTableMetaClient; -import com.uber.hoodie.common.table.HoodieTimeline; -import com.uber.hoodie.common.table.TableFileSystemView; -import com.uber.hoodie.common.table.timeline.HoodieInstant; -import com.uber.hoodie.common.table.view.HoodieTableFileSystemView; -import com.uber.hoodie.exception.DatasetNotFoundException; -import com.uber.hoodie.exception.HoodieIOException; -import com.uber.hoodie.exception.InvalidDatasetException; import java.io.IOException; import java.util.ArrayList; import java.util.HashMap; @@ -49,6 +39,16 @@ import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.mapred.RecordReader; import org.apache.hadoop.mapred.Reporter; import org.apache.hadoop.mapreduce.Job; +import org.apache.hudi.common.model.HoodieDataFile; +import org.apache.hudi.common.model.HoodiePartitionMetadata; +import org.apache.hudi.common.table.HoodieTableMetaClient; +import org.apache.hudi.common.table.HoodieTimeline; +import org.apache.hudi.common.table.TableFileSystemView.ReadOptimizedView; +import org.apache.hudi.common.table.timeline.HoodieInstant; +import org.apache.hudi.common.table.view.HoodieTableFileSystemView; +import org.apache.hudi.exception.DatasetNotFoundException; +import org.apache.hudi.exception.HoodieIOException; +import org.apache.hudi.exception.InvalidDatasetException; /** * HoodieInputFormat which understands the Hoodie File Structure and filters files based on the @@ -88,7 +88,7 @@ public class HoodieInputFormat extends MapredParquetInputFormat implements Confi // today HoodieTimeline timeline = metadata.getActiveTimeline().getCommitsTimeline() .filterCompletedInstants(); - TableFileSystemView.ReadOptimizedView roView = new HoodieTableFileSystemView(metadata, + ReadOptimizedView roView = new HoodieTableFileSystemView(metadata, timeline, statuses); if (HoodieHiveUtil.INCREMENTAL_SCAN_MODE.equals(mode)) { diff --git a/hoodie-hadoop-mr/src/main/java/com/uber/hoodie/hadoop/HoodieROTablePathFilter.java b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/HoodieROTablePathFilter.java similarity index 93% rename from hoodie-hadoop-mr/src/main/java/com/uber/hoodie/hadoop/HoodieROTablePathFilter.java rename to hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/HoodieROTablePathFilter.java index 806cba00b..f268810ca 100644 --- a/hoodie-hadoop-mr/src/main/java/com/uber/hoodie/hadoop/HoodieROTablePathFilter.java +++ b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/HoodieROTablePathFilter.java @@ -16,14 +16,8 @@ * limitations under the License. */ -package com.uber.hoodie.hadoop; +package org.apache.hudi.hadoop; -import com.uber.hoodie.common.model.HoodieDataFile; -import com.uber.hoodie.common.model.HoodiePartitionMetadata; -import com.uber.hoodie.common.table.HoodieTableMetaClient; -import com.uber.hoodie.common.table.view.HoodieTableFileSystemView; -import com.uber.hoodie.exception.DatasetNotFoundException; -import com.uber.hoodie.exception.HoodieException; import java.io.Serializable; import java.util.HashMap; import java.util.HashSet; @@ -35,6 +29,12 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.PathFilter; +import org.apache.hudi.common.model.HoodieDataFile; +import org.apache.hudi.common.model.HoodiePartitionMetadata; +import org.apache.hudi.common.table.HoodieTableMetaClient; +import org.apache.hudi.common.table.view.HoodieTableFileSystemView; +import org.apache.hudi.exception.DatasetNotFoundException; +import org.apache.hudi.exception.HoodieException; /** * Given a path is a part of - Hoodie dataset = accepts ONLY the latest version of each path - @@ -43,7 +43,7 @@ import org.apache.hadoop.fs.PathFilter; * We can set this filter, on a query engine's Hadoop Config and if it respects path filters, then * you should be able to query both hoodie and non-hoodie datasets as you would normally do. *

- * hadoopConf.setClass("mapreduce.input.pathFilter.class", com.uber.hoodie.hadoop + * hadoopConf.setClass("mapreduce.input.pathFilter.class", org.apache.hudi.hadoop * .HoodieROTablePathFilter.class, org.apache.hadoop.fs.PathFilter.class) */ public class HoodieROTablePathFilter implements PathFilter, Serializable { diff --git a/hoodie-hadoop-mr/src/main/java/com/uber/hoodie/hadoop/RecordReaderValueIterator.java b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/RecordReaderValueIterator.java similarity index 96% rename from hoodie-hadoop-mr/src/main/java/com/uber/hoodie/hadoop/RecordReaderValueIterator.java rename to hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/RecordReaderValueIterator.java index 28e7e5711..f8c231a7e 100644 --- a/hoodie-hadoop-mr/src/main/java/com/uber/hoodie/hadoop/RecordReaderValueIterator.java +++ b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/RecordReaderValueIterator.java @@ -16,15 +16,15 @@ * limitations under the License. */ -package com.uber.hoodie.hadoop; +package org.apache.hudi.hadoop; -import com.uber.hoodie.exception.HoodieException; import java.io.IOException; import java.util.Iterator; import java.util.NoSuchElementException; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.mapred.RecordReader; +import org.apache.hudi.exception.HoodieException; /** * Provides Iterator Interface to iterate value entries read from record reader diff --git a/hoodie-hadoop-mr/src/main/java/com/uber/hoodie/hadoop/SafeParquetRecordReaderWrapper.java b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/SafeParquetRecordReaderWrapper.java similarity index 99% rename from hoodie-hadoop-mr/src/main/java/com/uber/hoodie/hadoop/SafeParquetRecordReaderWrapper.java rename to hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/SafeParquetRecordReaderWrapper.java index 274e955a0..d35c6be25 100644 --- a/hoodie-hadoop-mr/src/main/java/com/uber/hoodie/hadoop/SafeParquetRecordReaderWrapper.java +++ b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/SafeParquetRecordReaderWrapper.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package com.uber.hoodie.hadoop; +package org.apache.hudi.hadoop; import java.io.IOException; import org.apache.hadoop.io.ArrayWritable; diff --git a/hoodie-hadoop-mr/src/main/java/com/uber/hoodie/hadoop/UseFileSplitsFromInputFormat.java b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/UseFileSplitsFromInputFormat.java similarity index 97% rename from hoodie-hadoop-mr/src/main/java/com/uber/hoodie/hadoop/UseFileSplitsFromInputFormat.java rename to hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/UseFileSplitsFromInputFormat.java index ac8c6b204..e3b59d968 100644 --- a/hoodie-hadoop-mr/src/main/java/com/uber/hoodie/hadoop/UseFileSplitsFromInputFormat.java +++ b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/UseFileSplitsFromInputFormat.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package com.uber.hoodie.hadoop; +package org.apache.hudi.hadoop; import java.lang.annotation.Documented; import java.lang.annotation.ElementType; diff --git a/hoodie-hadoop-mr/src/main/java/com/uber/hoodie/hadoop/hive/HoodieCombineHiveInputFormat.java b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/hive/HoodieCombineHiveInputFormat.java similarity index 99% rename from hoodie-hadoop-mr/src/main/java/com/uber/hoodie/hadoop/hive/HoodieCombineHiveInputFormat.java rename to hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/hive/HoodieCombineHiveInputFormat.java index 5f2fb47d3..06ae8d6aa 100644 --- a/hoodie-hadoop-mr/src/main/java/com/uber/hoodie/hadoop/hive/HoodieCombineHiveInputFormat.java +++ b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/hive/HoodieCombineHiveInputFormat.java @@ -16,11 +16,9 @@ * limitations under the License. */ -package com.uber.hoodie.hadoop.hive; +package org.apache.hudi.hadoop.hive; import com.google.common.annotations.VisibleForTesting; -import com.uber.hoodie.hadoop.HoodieInputFormat; -import com.uber.hoodie.hadoop.realtime.HoodieRealtimeInputFormat; import java.io.DataInput; import java.io.DataOutput; import java.io.IOException; @@ -72,6 +70,8 @@ import org.apache.hadoop.mapred.Reporter; import org.apache.hadoop.mapred.lib.CombineFileInputFormat; import org.apache.hadoop.mapred.lib.CombineFileSplit; import org.apache.hadoop.mapreduce.JobContext; +import org.apache.hudi.hadoop.HoodieInputFormat; +import org.apache.hudi.hadoop.realtime.HoodieRealtimeInputFormat; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -930,4 +930,4 @@ public class HoodieCombineHiveInputFormat { diff --git a/hoodie-hadoop-mr/src/main/java/com/uber/hoodie/hadoop/realtime/RealtimeUnmergedRecordReader.java b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/realtime/RealtimeUnmergedRecordReader.java similarity index 89% rename from hoodie-hadoop-mr/src/main/java/com/uber/hoodie/hadoop/realtime/RealtimeUnmergedRecordReader.java rename to hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/realtime/RealtimeUnmergedRecordReader.java index 5e5382d60..82f81b90e 100644 --- a/hoodie-hadoop-mr/src/main/java/com/uber/hoodie/hadoop/realtime/RealtimeUnmergedRecordReader.java +++ b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/realtime/RealtimeUnmergedRecordReader.java @@ -16,18 +16,8 @@ * limitations under the License. */ -package com.uber.hoodie.hadoop.realtime; +package org.apache.hudi.hadoop.realtime; -import com.uber.hoodie.common.table.log.HoodieUnMergedLogRecordScanner; -import com.uber.hoodie.common.util.DefaultSizeEstimator; -import com.uber.hoodie.common.util.FSUtils; -import com.uber.hoodie.common.util.Option; -import com.uber.hoodie.common.util.queue.BoundedInMemoryExecutor; -import com.uber.hoodie.common.util.queue.BoundedInMemoryQueueProducer; -import com.uber.hoodie.common.util.queue.FunctionBasedQueueProducer; -import com.uber.hoodie.common.util.queue.IteratorBasedQueueProducer; -import com.uber.hoodie.hadoop.RecordReaderValueIterator; -import com.uber.hoodie.hadoop.SafeParquetRecordReaderWrapper; import java.io.IOException; import java.util.ArrayList; import java.util.Iterator; @@ -37,6 +27,16 @@ import org.apache.hadoop.io.ArrayWritable; import org.apache.hadoop.io.NullWritable; import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.mapred.RecordReader; +import org.apache.hudi.common.table.log.HoodieUnMergedLogRecordScanner; +import org.apache.hudi.common.util.DefaultSizeEstimator; +import org.apache.hudi.common.util.FSUtils; +import org.apache.hudi.common.util.Option; +import org.apache.hudi.common.util.queue.BoundedInMemoryExecutor; +import org.apache.hudi.common.util.queue.BoundedInMemoryQueueProducer; +import org.apache.hudi.common.util.queue.FunctionBasedQueueProducer; +import org.apache.hudi.common.util.queue.IteratorBasedQueueProducer; +import org.apache.hudi.hadoop.RecordReaderValueIterator; +import org.apache.hudi.hadoop.SafeParquetRecordReaderWrapper; class RealtimeUnmergedRecordReader extends AbstractRealtimeRecordReader implements RecordReader { diff --git a/hoodie-hadoop-mr/src/main/resources/META-INF/LICENSE.txt b/hudi-hadoop-mr/src/main/resources/META-INF/LICENSE.txt similarity index 100% rename from hoodie-hadoop-mr/src/main/resources/META-INF/LICENSE.txt rename to hudi-hadoop-mr/src/main/resources/META-INF/LICENSE.txt diff --git a/hoodie-hadoop-mr/src/main/resources/META-INF/NOTICE.txt b/hudi-hadoop-mr/src/main/resources/META-INF/NOTICE.txt similarity index 100% rename from hoodie-hadoop-mr/src/main/resources/META-INF/NOTICE.txt rename to hudi-hadoop-mr/src/main/resources/META-INF/NOTICE.txt diff --git a/hoodie-hadoop-mr/src/test/java/com/uber/hoodie/hadoop/AnnotationTest.java b/hudi-hadoop-mr/src/test/java/org/apache/hudi/hadoop/AnnotationTest.java similarity index 97% rename from hoodie-hadoop-mr/src/test/java/com/uber/hoodie/hadoop/AnnotationTest.java rename to hudi-hadoop-mr/src/test/java/org/apache/hudi/hadoop/AnnotationTest.java index 55ab4712a..765b19ebb 100644 --- a/hoodie-hadoop-mr/src/test/java/com/uber/hoodie/hadoop/AnnotationTest.java +++ b/hudi-hadoop-mr/src/test/java/org/apache/hudi/hadoop/AnnotationTest.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package com.uber.hoodie.hadoop; +package org.apache.hudi.hadoop; import static org.junit.Assert.assertTrue; diff --git a/hoodie-hadoop-mr/src/test/java/com/uber/hoodie/hadoop/HoodieInputFormatTest.java b/hudi-hadoop-mr/src/test/java/org/apache/hudi/hadoop/HoodieInputFormatTest.java similarity index 99% rename from hoodie-hadoop-mr/src/test/java/com/uber/hoodie/hadoop/HoodieInputFormatTest.java rename to hudi-hadoop-mr/src/test/java/org/apache/hudi/hadoop/HoodieInputFormatTest.java index dbda1ed50..e5c79a928 100644 --- a/hoodie-hadoop-mr/src/test/java/com/uber/hoodie/hadoop/HoodieInputFormatTest.java +++ b/hudi-hadoop-mr/src/test/java/org/apache/hudi/hadoop/HoodieInputFormatTest.java @@ -16,11 +16,10 @@ * limitations under the License. */ -package com.uber.hoodie.hadoop; +package org.apache.hudi.hadoop; import static org.junit.Assert.assertEquals; -import com.uber.hoodie.common.util.FSUtils; import java.io.File; import java.io.IOException; import org.apache.avro.Schema; @@ -31,6 +30,7 @@ import org.apache.hadoop.mapred.FileInputFormat; import org.apache.hadoop.mapred.InputSplit; import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.mapred.RecordReader; +import org.apache.hudi.common.util.FSUtils; import org.junit.Before; import org.junit.Rule; import org.junit.Test; diff --git a/hoodie-hadoop-mr/src/test/java/com/uber/hoodie/hadoop/InputFormatTestUtil.java b/hudi-hadoop-mr/src/test/java/org/apache/hudi/hadoop/InputFormatTestUtil.java similarity index 96% rename from hoodie-hadoop-mr/src/test/java/com/uber/hoodie/hadoop/InputFormatTestUtil.java rename to hudi-hadoop-mr/src/test/java/org/apache/hudi/hadoop/InputFormatTestUtil.java index 2fa07055e..5543193b3 100644 --- a/hoodie-hadoop-mr/src/test/java/com/uber/hoodie/hadoop/InputFormatTestUtil.java +++ b/hudi-hadoop-mr/src/test/java/org/apache/hudi/hadoop/InputFormatTestUtil.java @@ -16,14 +16,8 @@ * limitations under the License. */ -package com.uber.hoodie.hadoop; +package org.apache.hudi.hadoop; -import com.uber.hoodie.common.model.HoodieRecord; -import com.uber.hoodie.common.model.HoodieTestUtils; -import com.uber.hoodie.common.table.timeline.HoodieActiveTimeline; -import com.uber.hoodie.common.util.FSUtils; -import com.uber.hoodie.common.util.HoodieAvroUtils; -import com.uber.hoodie.common.util.SchemaTestUtil; import java.io.File; import java.io.FilenameFilter; import java.io.IOException; @@ -37,6 +31,12 @@ import org.apache.avro.generic.GenericRecord; import org.apache.avro.generic.IndexedRecord; import org.apache.hadoop.fs.Path; import org.apache.hadoop.mapred.JobConf; +import org.apache.hudi.common.model.HoodieRecord; +import org.apache.hudi.common.model.HoodieTestUtils; +import org.apache.hudi.common.table.timeline.HoodieActiveTimeline; +import org.apache.hudi.common.util.FSUtils; +import org.apache.hudi.common.util.HoodieAvroUtils; +import org.apache.hudi.common.util.SchemaTestUtil; import org.apache.parquet.avro.AvroParquetWriter; import org.junit.rules.TemporaryFolder; diff --git a/hoodie-hadoop-mr/src/test/java/com/uber/hoodie/hadoop/TestHoodieROTablePathFilter.java b/hudi-hadoop-mr/src/test/java/org/apache/hudi/hadoop/TestHoodieROTablePathFilter.java similarity index 96% rename from hoodie-hadoop-mr/src/test/java/com/uber/hoodie/hadoop/TestHoodieROTablePathFilter.java rename to hudi-hadoop-mr/src/test/java/org/apache/hudi/hadoop/TestHoodieROTablePathFilter.java index af98a6e70..e085c639f 100644 --- a/hoodie-hadoop-mr/src/test/java/com/uber/hoodie/hadoop/TestHoodieROTablePathFilter.java +++ b/hudi-hadoop-mr/src/test/java/org/apache/hudi/hadoop/TestHoodieROTablePathFilter.java @@ -16,17 +16,17 @@ * limitations under the License. */ -package com.uber.hoodie.hadoop; +package org.apache.hudi.hadoop; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; -import com.uber.hoodie.common.model.HoodieTestUtils; -import com.uber.hoodie.common.table.HoodieTableMetaClient; import java.io.File; import java.io.IOException; import java.util.ArrayList; import org.apache.hadoop.fs.Path; +import org.apache.hudi.common.model.HoodieTestUtils; +import org.apache.hudi.common.table.HoodieTableMetaClient; import org.junit.Before; import org.junit.Rule; import org.junit.Test; diff --git a/hoodie-hadoop-mr/src/test/java/com/uber/hoodie/hadoop/TestRecordReaderValueIterator.java b/hudi-hadoop-mr/src/test/java/org/apache/hudi/hadoop/TestRecordReaderValueIterator.java similarity index 97% rename from hoodie-hadoop-mr/src/test/java/com/uber/hoodie/hadoop/TestRecordReaderValueIterator.java rename to hudi-hadoop-mr/src/test/java/org/apache/hudi/hadoop/TestRecordReaderValueIterator.java index 3a20a10af..c5f9b824b 100644 --- a/hoodie-hadoop-mr/src/test/java/com/uber/hoodie/hadoop/TestRecordReaderValueIterator.java +++ b/hudi-hadoop-mr/src/test/java/org/apache/hudi/hadoop/TestRecordReaderValueIterator.java @@ -16,9 +16,8 @@ * limitations under the License. */ -package com.uber.hoodie.hadoop; +package org.apache.hudi.hadoop; -import com.uber.hoodie.common.util.collection.Pair; import java.io.IOException; import java.util.List; import java.util.stream.Collectors; @@ -26,6 +25,7 @@ import java.util.stream.IntStream; import org.apache.hadoop.io.IntWritable; import org.apache.hadoop.io.Text; import org.apache.hadoop.mapred.RecordReader; +import org.apache.hudi.common.util.collection.Pair; import org.junit.Assert; import org.junit.Test; diff --git a/hoodie-hadoop-mr/src/test/java/com/uber/hoodie/hadoop/realtime/HoodieRealtimeRecordReaderTest.java b/hudi-hadoop-mr/src/test/java/org/apache/hudi/hadoop/realtime/HoodieRealtimeRecordReaderTest.java similarity index 95% rename from hoodie-hadoop-mr/src/test/java/com/uber/hoodie/hadoop/realtime/HoodieRealtimeRecordReaderTest.java rename to hudi-hadoop-mr/src/test/java/org/apache/hudi/hadoop/realtime/HoodieRealtimeRecordReaderTest.java index dc268d68c..a583a346e 100644 --- a/hoodie-hadoop-mr/src/test/java/com/uber/hoodie/hadoop/realtime/HoodieRealtimeRecordReaderTest.java +++ b/hudi-hadoop-mr/src/test/java/org/apache/hudi/hadoop/realtime/HoodieRealtimeRecordReaderTest.java @@ -16,28 +16,11 @@ * limitations under the License. */ -package com.uber.hoodie.hadoop.realtime; +package org.apache.hudi.hadoop.realtime; import static org.junit.Assert.assertTrue; import com.google.common.collect.Maps; -import com.uber.hoodie.common.model.FileSlice; -import com.uber.hoodie.common.model.HoodieLogFile; -import com.uber.hoodie.common.model.HoodieTableType; -import com.uber.hoodie.common.model.HoodieTestUtils; -import com.uber.hoodie.common.table.HoodieTimeline; -import com.uber.hoodie.common.table.log.HoodieLogFormat; -import com.uber.hoodie.common.table.log.block.HoodieAvroDataBlock; -import com.uber.hoodie.common.table.log.block.HoodieCommandBlock; -import com.uber.hoodie.common.table.log.block.HoodieCommandBlock.HoodieCommandBlockTypeEnum; -import com.uber.hoodie.common.table.log.block.HoodieLogBlock; -import com.uber.hoodie.common.table.log.block.HoodieLogBlock.HeaderMetadataType; -import com.uber.hoodie.common.util.FSUtils; -import com.uber.hoodie.common.util.HoodieAvroUtils; -import com.uber.hoodie.common.util.SchemaTestUtil; -import com.uber.hoodie.common.util.collection.Pair; -import com.uber.hoodie.exception.HoodieException; -import com.uber.hoodie.hadoop.InputFormatTestUtil; import java.io.File; import java.io.IOException; import java.util.ArrayList; @@ -67,6 +50,24 @@ import org.apache.hadoop.mapred.FileInputFormat; import org.apache.hadoop.mapred.FileSplit; import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.mapred.RecordReader; +import org.apache.hudi.common.model.FileSlice; +import org.apache.hudi.common.model.HoodieLogFile; +import org.apache.hudi.common.model.HoodieTableType; +import org.apache.hudi.common.model.HoodieTestUtils; +import org.apache.hudi.common.table.HoodieTimeline; +import org.apache.hudi.common.table.log.HoodieLogFormat; +import org.apache.hudi.common.table.log.HoodieLogFormat.Writer; +import org.apache.hudi.common.table.log.block.HoodieAvroDataBlock; +import org.apache.hudi.common.table.log.block.HoodieCommandBlock; +import org.apache.hudi.common.table.log.block.HoodieCommandBlock.HoodieCommandBlockTypeEnum; +import org.apache.hudi.common.table.log.block.HoodieLogBlock; +import org.apache.hudi.common.table.log.block.HoodieLogBlock.HeaderMetadataType; +import org.apache.hudi.common.util.FSUtils; +import org.apache.hudi.common.util.HoodieAvroUtils; +import org.apache.hudi.common.util.SchemaTestUtil; +import org.apache.hudi.common.util.collection.Pair; +import org.apache.hudi.exception.HoodieException; +import org.apache.hudi.hadoop.InputFormatTestUtil; import org.junit.Assert; import org.junit.Before; import org.junit.Rule; @@ -90,16 +91,16 @@ public class HoodieRealtimeRecordReaderTest { @Rule public TemporaryFolder basePath = new TemporaryFolder(); - private HoodieLogFormat.Writer writeLogFile(File partitionDir, Schema schema, String fileId, + private Writer writeLogFile(File partitionDir, Schema schema, String fileId, String baseCommit, String newCommit, int numberOfRecords) throws InterruptedException, IOException { return writeDataBlockToLogFile(partitionDir, schema, fileId, baseCommit, newCommit, numberOfRecords, 0, 0); } - private HoodieLogFormat.Writer writeRollback(File partitionDir, Schema schema, String fileId, + private Writer writeRollback(File partitionDir, Schema schema, String fileId, String baseCommit, String newCommit, String rolledBackInstant, int logVersion) throws InterruptedException, IOException { - HoodieLogFormat.Writer writer = HoodieLogFormat.newWriterBuilder() + Writer writer = HoodieLogFormat.newWriterBuilder() .onParentPath(new Path(partitionDir.getPath())) .withFileId(fileId).overBaseCommit(baseCommit) .withFs(fs) @@ -131,7 +132,7 @@ public class HoodieRealtimeRecordReaderTest { records.add(SchemaTestUtil.generateAvroRecordFromJson(schema, i, newCommit, "fileid0")); } Schema writeSchema = records.get(0).getSchema(); - Map header = Maps.newHashMap(); + Map header = Maps.newHashMap(); header.put(HoodieLogBlock.HeaderMetadataType.INSTANT_TIME, newCommit); header.put(HoodieLogBlock.HeaderMetadataType.SCHEMA, writeSchema.toString()); HoodieAvroDataBlock dataBlock = new HoodieAvroDataBlock(records, header); @@ -567,4 +568,4 @@ public class HoodieRealtimeRecordReaderTest { // keep reading } } -} \ No newline at end of file +} diff --git a/hoodie-hadoop-mr/src/test/resources/log4j-surefire-quiet.properties b/hudi-hadoop-mr/src/test/resources/log4j-surefire-quiet.properties similarity index 100% rename from hoodie-hadoop-mr/src/test/resources/log4j-surefire-quiet.properties rename to hudi-hadoop-mr/src/test/resources/log4j-surefire-quiet.properties diff --git a/hoodie-hadoop-mr/src/test/resources/log4j-surefire.properties b/hudi-hadoop-mr/src/test/resources/log4j-surefire.properties similarity index 100% rename from hoodie-hadoop-mr/src/test/resources/log4j-surefire.properties rename to hudi-hadoop-mr/src/test/resources/log4j-surefire.properties diff --git a/hoodie-hadoop-mr/src/test/resources/sample1.avsc b/hudi-hadoop-mr/src/test/resources/sample1.avsc similarity index 99% rename from hoodie-hadoop-mr/src/test/resources/sample1.avsc rename to hudi-hadoop-mr/src/test/resources/sample1.avsc index 6d8e9010f..20455fa5a 100644 --- a/hoodie-hadoop-mr/src/test/resources/sample1.avsc +++ b/hudi-hadoop-mr/src/test/resources/sample1.avsc @@ -14,4 +14,4 @@ "name" : "_hoodie_commit_seqno", "type" : "string" }] -} \ No newline at end of file +} diff --git a/hoodie-hive/pom.xml b/hudi-hive/pom.xml similarity index 93% rename from hoodie-hive/pom.xml rename to hudi-hive/pom.xml index 88433537f..b9f6e01f7 100644 --- a/hoodie-hive/pom.xml +++ b/hudi-hive/pom.xml @@ -17,13 +17,13 @@ --> - hoodie - com.uber.hoodie - 0.4.8-SNAPSHOT + hudi + org.apache.hudi + 0.5.0-SNAPSHOT 4.0.0 - hoodie-hive + hudi-hive jar @@ -170,18 +170,18 @@ - com.uber.hoodie - hoodie-hadoop-mr + org.apache.hudi + hudi-hadoop-mr ${project.version} - com.uber.hoodie - hoodie-common + org.apache.hudi + hudi-common ${project.version} - com.uber.hoodie - hoodie-common + org.apache.hudi + hudi-common ${project.version} tests test diff --git a/hoodie-hive/run_sync_tool.sh b/hudi-hive/run_sync_tool.sh similarity index 84% rename from hoodie-hive/run_sync_tool.sh rename to hudi-hive/run_sync_tool.sh index fb6e24fe7..0a39c0984 100755 --- a/hoodie-hive/run_sync_tool.sh +++ b/hudi-hive/run_sync_tool.sh @@ -15,7 +15,7 @@ fi DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" #Ensure we pick the right jar even for hive11 builds -HOODIE_HIVE_UBER_JAR=`ls -c $DIR/../packaging/hoodie-hive-bundle/target/hoodie-hive-*.jar | grep -v source | head -1` +HUDI_HIVE_UBER_JAR=`ls -c $DIR/../packaging/hudi-hive-bundle/target/hudi-hive-*.jar | grep -v source | head -1` if [ -z "$HADOOP_CONF_DIR" ]; then echo "setting hadoop conf dir" @@ -35,5 +35,5 @@ HIVE_JARS=$HIVE_METASTORE:$HIVE_SERVICE:$HIVE_EXEC:$HIVE_SERVICE:$HIVE_JDBC HADOOP_HIVE_JARS=${HIVE_JARS}:${HADOOP_HOME}/share/hadoop/common/*:${HADOOP_HOME}/share/hadoop/mapreduce/*:${HADOOP_HOME}/share/hadoop/hdfs/*:${HADOOP_HOME}/share/hadoop/common/lib/*:${HADOOP_HOME}/share/hadoop/hdfs/lib/* -echo "Running Command : java -cp ${HADOOP_HIVE_JARS}:${HADOOP_CONF_DIR}:$HOODIE_HIVE_UBER_JAR com.uber.hoodie.hive.HiveSyncTool $@" -java -cp $HOODIE_HIVE_UBER_JAR:${HADOOP_HIVE_JARS}:${HADOOP_CONF_DIR} com.uber.hoodie.hive.HiveSyncTool "$@" +echo "Running Command : java -cp ${HADOOP_HIVE_JARS}:${HADOOP_CONF_DIR}:$HUDI_HIVE_UBER_JAR org.apache.hudi.hive.HiveSyncTool $@" +java -cp $HUDI_HIVE_UBER_JAR:${HADOOP_HIVE_JARS}:${HADOOP_CONF_DIR} org.apache.hudi.hive.HiveSyncTool "$@" diff --git a/hoodie-hive/src/assembly/src.xml b/hudi-hive/src/assembly/src.xml similarity index 100% rename from hoodie-hive/src/assembly/src.xml rename to hudi-hive/src/assembly/src.xml diff --git a/hoodie-hive/src/main/java/com/uber/hoodie/hive/HiveSyncConfig.java b/hudi-hive/src/main/java/org/apache/hudi/hive/HiveSyncConfig.java similarity index 87% rename from hoodie-hive/src/main/java/com/uber/hoodie/hive/HiveSyncConfig.java rename to hudi-hive/src/main/java/org/apache/hudi/hive/HiveSyncConfig.java index 9748c21bb..829616371 100644 --- a/hoodie-hive/src/main/java/com/uber/hoodie/hive/HiveSyncConfig.java +++ b/hudi-hive/src/main/java/org/apache/hudi/hive/HiveSyncConfig.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package com.uber.hoodie.hive; +package org.apache.hudi.hive; import com.beust.jcommander.Parameter; import java.io.Serializable; @@ -67,6 +67,13 @@ public class HiveSyncConfig implements Serializable { + "not set this parameter") public Boolean assumeDatePartitioning = false; + @Parameter(names = { + "--use-pre-apache-input-format"}, description = "Use InputFormat under com.uber.hoodie package " + + "instead of org.apache.hudi package. Use this when you are in the process of migrating from " + + "com.uber.hoodie to org.apache.hudi. Stop using this after you migrated the table definition to " + + "org.apache.hudi input format.") + public Boolean usePreApacheInputFormat = false; + @Parameter(names = {"--help", "-h"}, help = true) public Boolean help = false; @@ -97,6 +104,7 @@ public class HiveSyncConfig implements Serializable { newConfig.partitionValueExtractorClass = cfg.partitionValueExtractorClass; newConfig.jdbcUrl = cfg.jdbcUrl; newConfig.tableName = cfg.tableName; + newConfig.usePreApacheInputFormat = cfg.usePreApacheInputFormat; return newConfig; } } diff --git a/hoodie-hive/src/main/java/com/uber/hoodie/hive/HiveSyncTool.java b/hudi-hive/src/main/java/org/apache/hudi/hive/HiveSyncTool.java similarity index 85% rename from hoodie-hive/src/main/java/com/uber/hoodie/hive/HiveSyncTool.java rename to hudi-hive/src/main/java/org/apache/hudi/hive/HiveSyncTool.java index 5412dab51..a70106419 100644 --- a/hoodie-hive/src/main/java/com/uber/hoodie/hive/HiveSyncTool.java +++ b/hudi-hive/src/main/java/org/apache/hudi/hive/HiveSyncTool.java @@ -16,17 +16,9 @@ * limitations under the License. */ -package com.uber.hoodie.hive; +package org.apache.hudi.hive; import com.beust.jcommander.JCommander; -import com.uber.hoodie.common.util.FSUtils; -import com.uber.hoodie.common.util.Option; -import com.uber.hoodie.exception.InvalidDatasetException; -import com.uber.hoodie.hadoop.HoodieInputFormat; -import com.uber.hoodie.hadoop.realtime.HoodieRealtimeInputFormat; -import com.uber.hoodie.hive.HoodieHiveClient.PartitionEvent; -import com.uber.hoodie.hive.HoodieHiveClient.PartitionEvent.PartitionEventType; -import com.uber.hoodie.hive.util.SchemaUtil; import java.util.List; import java.util.Map; import java.util.stream.Collectors; @@ -36,6 +28,14 @@ import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.api.Partition; import org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat; import org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe; +import org.apache.hudi.common.util.FSUtils; +import org.apache.hudi.common.util.Option; +import org.apache.hudi.exception.InvalidDatasetException; +import org.apache.hudi.hadoop.HoodieInputFormat; +import org.apache.hudi.hadoop.realtime.HoodieRealtimeInputFormat; +import org.apache.hudi.hive.HoodieHiveClient.PartitionEvent; +import org.apache.hudi.hive.HoodieHiveClient.PartitionEvent.PartitionEventType; +import org.apache.hudi.hive.util.SchemaUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import parquet.schema.MessageType; @@ -62,7 +62,7 @@ public class HiveSyncTool { this.cfg = cfg; } - public void syncHoodieTable() { + public void syncHoodieTable() throws ClassNotFoundException { switch (hoodieHiveClient.getTableType()) { case COPY_ON_WRITE: syncHoodieTable(false); @@ -84,7 +84,7 @@ public class HiveSyncTool { hoodieHiveClient.close(); } - private void syncHoodieTable(boolean isRealTime) { + private void syncHoodieTable(boolean isRealTime) throws ClassNotFoundException { LOG.info("Trying to sync hoodie table " + cfg.tableName + " with base path " + hoodieHiveClient.getBasePath() + " of type " + hoodieHiveClient.getTableType()); @@ -117,22 +117,28 @@ public class HiveSyncTool { * If not, evolves the table schema. * * @param tableExists - does table exist - * @param schema - extracted schema + * @param schema - extracted schema */ - private void syncSchema(boolean tableExists, boolean isRealTime, MessageType schema) { + private void syncSchema(boolean tableExists, boolean isRealTime, MessageType schema) throws ClassNotFoundException { // Check and sync schema if (!tableExists) { LOG.info("Table " + cfg.tableName + " is not found. Creating it"); if (!isRealTime) { // TODO - RO Table for MOR only after major compaction (UnboundedCompaction is default // for now) - hoodieHiveClient.createTable(schema, HoodieInputFormat.class.getName(), + String inputFormatClassName = + cfg.usePreApacheInputFormat ? com.uber.hoodie.hadoop.HoodieInputFormat.class.getName() + : HoodieInputFormat.class.getName(); + hoodieHiveClient.createTable(schema, inputFormatClassName, MapredParquetOutputFormat.class.getName(), ParquetHiveSerDe.class.getName()); } else { // Custom serde will not work with ALTER TABLE REPLACE COLUMNS // https://github.com/apache/hive/blob/release-1.1.0/ql/src/java/org/apache/hadoop/hive // /ql/exec/DDLTask.java#L3488 - hoodieHiveClient.createTable(schema, HoodieRealtimeInputFormat.class.getName(), + String inputFormatClassName = + cfg.usePreApacheInputFormat ? com.uber.hoodie.hadoop.realtime.HoodieRealtimeInputFormat.class.getName() + : HoodieRealtimeInputFormat.class.getName(); + hoodieHiveClient.createTable(schema, inputFormatClassName, MapredParquetOutputFormat.class.getName(), ParquetHiveSerDe.class.getName()); } } else { diff --git a/hoodie-hive/src/main/java/com/uber/hoodie/hive/HoodieHiveClient.java b/hudi-hive/src/main/java/org/apache/hudi/hive/HoodieHiveClient.java similarity index 97% rename from hoodie-hive/src/main/java/com/uber/hoodie/hive/HoodieHiveClient.java rename to hudi-hive/src/main/java/org/apache/hudi/hive/HoodieHiveClient.java index 16f42ed90..90dded031 100644 --- a/hoodie-hive/src/main/java/com/uber/hoodie/hive/HoodieHiveClient.java +++ b/hudi-hive/src/main/java/org/apache/hudi/hive/HoodieHiveClient.java @@ -16,24 +16,11 @@ * limitations under the License. */ -package com.uber.hoodie.hive; +package org.apache.hudi.hive; import com.google.common.base.Preconditions; import com.google.common.collect.Lists; import com.google.common.collect.Maps; -import com.uber.hoodie.common.model.HoodieCommitMetadata; -import com.uber.hoodie.common.model.HoodieFileFormat; -import com.uber.hoodie.common.model.HoodieLogFile; -import com.uber.hoodie.common.model.HoodieTableType; -import com.uber.hoodie.common.table.HoodieTableMetaClient; -import com.uber.hoodie.common.table.HoodieTimeline; -import com.uber.hoodie.common.table.timeline.HoodieInstant; -import com.uber.hoodie.common.util.FSUtils; -import com.uber.hoodie.common.util.Option; -import com.uber.hoodie.common.util.collection.Pair; -import com.uber.hoodie.exception.HoodieIOException; -import com.uber.hoodie.exception.InvalidDatasetException; -import com.uber.hoodie.hive.util.SchemaUtil; import java.io.IOException; import java.sql.Connection; import java.sql.DatabaseMetaData; @@ -57,6 +44,19 @@ import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.metastore.api.Partition; import org.apache.hadoop.hive.metastore.api.Table; import org.apache.hive.jdbc.HiveDriver; +import org.apache.hudi.common.model.HoodieCommitMetadata; +import org.apache.hudi.common.model.HoodieFileFormat; +import org.apache.hudi.common.model.HoodieLogFile; +import org.apache.hudi.common.model.HoodieTableType; +import org.apache.hudi.common.table.HoodieTableMetaClient; +import org.apache.hudi.common.table.HoodieTimeline; +import org.apache.hudi.common.table.timeline.HoodieInstant; +import org.apache.hudi.common.util.FSUtils; +import org.apache.hudi.common.util.Option; +import org.apache.hudi.common.util.collection.Pair; +import org.apache.hudi.exception.HoodieIOException; +import org.apache.hudi.exception.InvalidDatasetException; +import org.apache.hudi.hive.util.SchemaUtil; import org.apache.thrift.TException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/hoodie-hive/src/main/java/com/uber/hoodie/hive/HoodieHiveSyncException.java b/hudi-hive/src/main/java/org/apache/hudi/hive/HoodieHiveSyncException.java similarity index 97% rename from hoodie-hive/src/main/java/com/uber/hoodie/hive/HoodieHiveSyncException.java rename to hudi-hive/src/main/java/org/apache/hudi/hive/HoodieHiveSyncException.java index cc57ce86d..d52ac71aa 100644 --- a/hoodie-hive/src/main/java/com/uber/hoodie/hive/HoodieHiveSyncException.java +++ b/hudi-hive/src/main/java/org/apache/hudi/hive/HoodieHiveSyncException.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package com.uber.hoodie.hive; +package org.apache.hudi.hive; public class HoodieHiveSyncException extends RuntimeException { diff --git a/hoodie-hive/src/main/java/com/uber/hoodie/hive/MultiPartKeysValueExtractor.java b/hudi-hive/src/main/java/org/apache/hudi/hive/MultiPartKeysValueExtractor.java similarity index 98% rename from hoodie-hive/src/main/java/com/uber/hoodie/hive/MultiPartKeysValueExtractor.java rename to hudi-hive/src/main/java/org/apache/hudi/hive/MultiPartKeysValueExtractor.java index 6a01ccaae..9701e5b4f 100644 --- a/hoodie-hive/src/main/java/com/uber/hoodie/hive/MultiPartKeysValueExtractor.java +++ b/hudi-hive/src/main/java/org/apache/hudi/hive/MultiPartKeysValueExtractor.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package com.uber.hoodie.hive; +package org.apache.hudi.hive; import com.google.common.base.Preconditions; import java.util.Arrays; diff --git a/hoodie-hive/src/main/java/com/uber/hoodie/hive/NonPartitionedExtractor.java b/hudi-hive/src/main/java/org/apache/hudi/hive/NonPartitionedExtractor.java similarity index 97% rename from hoodie-hive/src/main/java/com/uber/hoodie/hive/NonPartitionedExtractor.java rename to hudi-hive/src/main/java/org/apache/hudi/hive/NonPartitionedExtractor.java index be8c7e578..860044e7a 100644 --- a/hoodie-hive/src/main/java/com/uber/hoodie/hive/NonPartitionedExtractor.java +++ b/hudi-hive/src/main/java/org/apache/hudi/hive/NonPartitionedExtractor.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package com.uber.hoodie.hive; +package org.apache.hudi.hive; import java.util.ArrayList; import java.util.List; diff --git a/hoodie-hive/src/main/java/com/uber/hoodie/hive/PartitionValueExtractor.java b/hudi-hive/src/main/java/org/apache/hudi/hive/PartitionValueExtractor.java similarity index 97% rename from hoodie-hive/src/main/java/com/uber/hoodie/hive/PartitionValueExtractor.java rename to hudi-hive/src/main/java/org/apache/hudi/hive/PartitionValueExtractor.java index b1886622a..76a44d723 100644 --- a/hoodie-hive/src/main/java/com/uber/hoodie/hive/PartitionValueExtractor.java +++ b/hudi-hive/src/main/java/org/apache/hudi/hive/PartitionValueExtractor.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package com.uber.hoodie.hive; +package org.apache.hudi.hive; import java.io.Serializable; import java.util.List; diff --git a/hoodie-hive/src/main/java/com/uber/hoodie/hive/SchemaDifference.java b/hudi-hive/src/main/java/org/apache/hudi/hive/SchemaDifference.java similarity index 99% rename from hoodie-hive/src/main/java/com/uber/hoodie/hive/SchemaDifference.java rename to hudi-hive/src/main/java/org/apache/hudi/hive/SchemaDifference.java index 1299e8b8c..8a7cee95d 100644 --- a/hoodie-hive/src/main/java/com/uber/hoodie/hive/SchemaDifference.java +++ b/hudi-hive/src/main/java/org/apache/hudi/hive/SchemaDifference.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package com.uber.hoodie.hive; +package org.apache.hudi.hive; import com.google.common.base.Objects; import com.google.common.collect.ImmutableList; diff --git a/hoodie-hive/src/main/java/com/uber/hoodie/hive/SlashEncodedDayPartitionValueExtractor.java b/hudi-hive/src/main/java/org/apache/hudi/hive/SlashEncodedDayPartitionValueExtractor.java similarity index 98% rename from hoodie-hive/src/main/java/com/uber/hoodie/hive/SlashEncodedDayPartitionValueExtractor.java rename to hudi-hive/src/main/java/org/apache/hudi/hive/SlashEncodedDayPartitionValueExtractor.java index 5906cfa46..1b772ab37 100644 --- a/hoodie-hive/src/main/java/com/uber/hoodie/hive/SlashEncodedDayPartitionValueExtractor.java +++ b/hudi-hive/src/main/java/org/apache/hudi/hive/SlashEncodedDayPartitionValueExtractor.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package com.uber.hoodie.hive; +package org.apache.hudi.hive; import com.beust.jcommander.internal.Lists; import java.util.List; diff --git a/hoodie-hive/src/main/java/com/uber/hoodie/hive/util/ColumnNameXLator.java b/hudi-hive/src/main/java/org/apache/hudi/hive/util/ColumnNameXLator.java similarity index 97% rename from hoodie-hive/src/main/java/com/uber/hoodie/hive/util/ColumnNameXLator.java rename to hudi-hive/src/main/java/org/apache/hudi/hive/util/ColumnNameXLator.java index a8643b307..a3584b108 100644 --- a/hoodie-hive/src/main/java/com/uber/hoodie/hive/util/ColumnNameXLator.java +++ b/hudi-hive/src/main/java/org/apache/hudi/hive/util/ColumnNameXLator.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package com.uber.hoodie.hive.util; +package org.apache.hudi.hive.util; import com.google.common.collect.Maps; import java.util.Iterator; diff --git a/hoodie-hive/src/main/java/com/uber/hoodie/hive/util/SchemaUtil.java b/hudi-hive/src/main/java/org/apache/hudi/hive/util/SchemaUtil.java similarity index 97% rename from hoodie-hive/src/main/java/com/uber/hoodie/hive/util/SchemaUtil.java rename to hudi-hive/src/main/java/org/apache/hudi/hive/util/SchemaUtil.java index 5da311598..69ea8d7f5 100644 --- a/hoodie-hive/src/main/java/com/uber/hoodie/hive/util/SchemaUtil.java +++ b/hudi-hive/src/main/java/org/apache/hudi/hive/util/SchemaUtil.java @@ -16,18 +16,10 @@ * limitations under the License. */ -package com.uber.hoodie.hive.util; +package org.apache.hudi.hive.util; import com.google.common.collect.Maps; import com.google.common.collect.Sets; -import com.uber.hoodie.common.model.HoodieLogFile; -import com.uber.hoodie.common.table.log.HoodieLogFormat; -import com.uber.hoodie.common.table.log.HoodieLogFormat.Reader; -import com.uber.hoodie.common.table.log.block.HoodieAvroDataBlock; -import com.uber.hoodie.common.table.log.block.HoodieLogBlock; -import com.uber.hoodie.hive.HiveSyncConfig; -import com.uber.hoodie.hive.HoodieHiveSyncException; -import com.uber.hoodie.hive.SchemaDifference; import java.io.IOException; import java.util.ArrayList; import java.util.List; @@ -36,6 +28,14 @@ import java.util.Set; import java.util.stream.Collectors; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hudi.common.model.HoodieLogFile; +import org.apache.hudi.common.table.log.HoodieLogFormat; +import org.apache.hudi.common.table.log.HoodieLogFormat.Reader; +import org.apache.hudi.common.table.log.block.HoodieAvroDataBlock; +import org.apache.hudi.common.table.log.block.HoodieLogBlock; +import org.apache.hudi.hive.HiveSyncConfig; +import org.apache.hudi.hive.HoodieHiveSyncException; +import org.apache.hudi.hive.SchemaDifference; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import parquet.schema.DecimalMetadata; diff --git a/hoodie-hive/src/main/resources/META-INF/LICENSE.txt b/hudi-hive/src/main/resources/META-INF/LICENSE.txt similarity index 100% rename from hoodie-hive/src/main/resources/META-INF/LICENSE.txt rename to hudi-hive/src/main/resources/META-INF/LICENSE.txt diff --git a/hoodie-hive/src/main/resources/META-INF/NOTICE.txt b/hudi-hive/src/main/resources/META-INF/NOTICE.txt similarity index 100% rename from hoodie-hive/src/main/resources/META-INF/NOTICE.txt rename to hudi-hive/src/main/resources/META-INF/NOTICE.txt diff --git a/hoodie-hive/src/test/java/com/uber/hoodie/hive/HiveSyncToolTest.java b/hudi-hive/src/test/java/org/apache/hudi/hive/HiveSyncToolTest.java similarity index 93% rename from hoodie-hive/src/test/java/com/uber/hoodie/hive/HiveSyncToolTest.java rename to hudi-hive/src/test/java/org/apache/hudi/hive/HiveSyncToolTest.java index 51dded52d..fe6e196be 100644 --- a/hoodie-hive/src/test/java/com/uber/hoodie/hive/HiveSyncToolTest.java +++ b/hudi-hive/src/test/java/org/apache/hudi/hive/HiveSyncToolTest.java @@ -16,27 +16,25 @@ * limitations under the License. */ -package com.uber.hoodie.hive; +package org.apache.hudi.hive; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; import com.google.common.collect.Lists; -import com.uber.hoodie.common.util.Option; -import com.uber.hoodie.common.util.SchemaTestUtil; -import com.uber.hoodie.hive.HoodieHiveClient.PartitionEvent; -import com.uber.hoodie.hive.HoodieHiveClient.PartitionEvent.PartitionEventType; -import com.uber.hoodie.hive.util.SchemaUtil; import java.io.IOException; import java.net.URISyntaxException; import java.util.List; import org.apache.hadoop.hive.metastore.api.Partition; -import org.apache.thrift.TException; +import org.apache.hudi.common.util.Option; +import org.apache.hudi.common.util.SchemaTestUtil; +import org.apache.hudi.hive.HoodieHiveClient.PartitionEvent; +import org.apache.hudi.hive.HoodieHiveClient.PartitionEvent.PartitionEventType; +import org.apache.hudi.hive.util.SchemaUtil; import org.joda.time.DateTime; import org.junit.Before; import org.junit.Test; -import org.junit.runners.model.InitializationError; import parquet.schema.MessageType; import parquet.schema.OriginalType; import parquet.schema.PrimitiveType; @@ -146,9 +144,7 @@ public class HiveSyncToolTest { @Test - public void testBasicSync() - throws IOException, InitializationError, URISyntaxException, TException, - InterruptedException { + public void testBasicSync() throws Exception { String commitTime = "100"; TestUtil.createCOWDataset(commitTime, 5); HoodieHiveClient hiveClient = new HoodieHiveClient(TestUtil.hiveSyncConfig, @@ -170,9 +166,7 @@ public class HiveSyncToolTest { } @Test - public void testSyncIncremental() - throws IOException, InitializationError, URISyntaxException, TException, - InterruptedException { + public void testSyncIncremental() throws Exception { String commitTime1 = "100"; TestUtil.createCOWDataset(commitTime1, 5); HoodieHiveClient hiveClient = new HoodieHiveClient(TestUtil.hiveSyncConfig, @@ -215,9 +209,7 @@ public class HiveSyncToolTest { } @Test - public void testSyncIncrementalWithSchemaEvolution() - throws IOException, InitializationError, URISyntaxException, TException, - InterruptedException { + public void testSyncIncrementalWithSchemaEvolution() throws Exception { String commitTime1 = "100"; TestUtil.createCOWDataset(commitTime1, 5); HoodieHiveClient hiveClient = new HoodieHiveClient(TestUtil.hiveSyncConfig, @@ -253,9 +245,7 @@ public class HiveSyncToolTest { } @Test - public void testSyncMergeOnRead() - throws IOException, InitializationError, URISyntaxException, TException, - InterruptedException { + public void testSyncMergeOnRead() throws Exception { String commitTime = "100"; String deltaCommitTime = "101"; TestUtil.createMORDataset(commitTime, deltaCommitTime, 5); @@ -303,8 +293,7 @@ public class HiveSyncToolTest { @Test public void testSyncMergeOnReadRT() - throws IOException, InitializationError, URISyntaxException, TException, - InterruptedException { + throws Exception { String commitTime = "100"; String deltaCommitTime = "101"; String roTablename = TestUtil.hiveSyncConfig.tableName; @@ -359,8 +348,7 @@ public class HiveSyncToolTest { @Test public void testMultiPartitionKeySync() - throws IOException, InitializationError, URISyntaxException, TException, - InterruptedException { + throws Exception { String commitTime = "100"; TestUtil.createCOWDataset(commitTime, 5); diff --git a/hoodie-hive/src/test/java/com/uber/hoodie/hive/TestUtil.java b/hudi-hive/src/test/java/org/apache/hudi/hive/TestUtil.java similarity index 92% rename from hoodie-hive/src/test/java/com/uber/hoodie/hive/TestUtil.java rename to hudi-hive/src/test/java/org/apache/hudi/hive/TestUtil.java index 41be9262b..6e9e06bdf 100644 --- a/hoodie-hive/src/test/java/com/uber/hoodie/hive/TestUtil.java +++ b/hudi-hive/src/test/java/org/apache/hudi/hive/TestUtil.java @@ -16,33 +16,13 @@ * limitations under the License. */ -package com.uber.hoodie.hive; +package org.apache.hudi.hive; import static org.junit.Assert.fail; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.collect.Sets; -import com.uber.hoodie.avro.HoodieAvroWriteSupport; -import com.uber.hoodie.common.BloomFilter; -import com.uber.hoodie.common.minicluster.HdfsTestService; -import com.uber.hoodie.common.minicluster.ZookeeperTestService; -import com.uber.hoodie.common.model.HoodieAvroPayload; -import com.uber.hoodie.common.model.HoodieCommitMetadata; -import com.uber.hoodie.common.model.HoodieDataFile; -import com.uber.hoodie.common.model.HoodieDeltaWriteStat; -import com.uber.hoodie.common.model.HoodieLogFile; -import com.uber.hoodie.common.model.HoodieTableType; -import com.uber.hoodie.common.model.HoodieWriteStat; -import com.uber.hoodie.common.table.HoodieTableMetaClient; -import com.uber.hoodie.common.table.HoodieTimeline; -import com.uber.hoodie.common.table.log.HoodieLogFormat; -import com.uber.hoodie.common.table.log.HoodieLogFormat.Writer; -import com.uber.hoodie.common.table.log.block.HoodieAvroDataBlock; -import com.uber.hoodie.common.table.log.block.HoodieLogBlock; -import com.uber.hoodie.common.util.FSUtils; -import com.uber.hoodie.common.util.SchemaTestUtil; -import com.uber.hoodie.hive.util.HiveTestService; import java.io.File; import java.io.IOException; import java.net.URISyntaxException; @@ -62,6 +42,27 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hive.service.server.HiveServer2; +import org.apache.hudi.avro.HoodieAvroWriteSupport; +import org.apache.hudi.common.BloomFilter; +import org.apache.hudi.common.minicluster.HdfsTestService; +import org.apache.hudi.common.minicluster.ZookeeperTestService; +import org.apache.hudi.common.model.HoodieAvroPayload; +import org.apache.hudi.common.model.HoodieCommitMetadata; +import org.apache.hudi.common.model.HoodieDataFile; +import org.apache.hudi.common.model.HoodieDeltaWriteStat; +import org.apache.hudi.common.model.HoodieLogFile; +import org.apache.hudi.common.model.HoodieTableType; +import org.apache.hudi.common.model.HoodieWriteStat; +import org.apache.hudi.common.table.HoodieTableMetaClient; +import org.apache.hudi.common.table.HoodieTimeline; +import org.apache.hudi.common.table.log.HoodieLogFormat; +import org.apache.hudi.common.table.log.HoodieLogFormat.Writer; +import org.apache.hudi.common.table.log.block.HoodieAvroDataBlock; +import org.apache.hudi.common.table.log.block.HoodieLogBlock; +import org.apache.hudi.common.table.log.block.HoodieLogBlock.HeaderMetadataType; +import org.apache.hudi.common.util.FSUtils; +import org.apache.hudi.common.util.SchemaTestUtil; +import org.apache.hudi.hive.util.HiveTestService; import org.apache.parquet.avro.AvroSchemaConverter; import org.apache.parquet.hadoop.ParquetWriter; import org.apache.parquet.hadoop.metadata.CompressionCodecName; @@ -108,6 +109,7 @@ public class TestUtil { hiveSyncConfig.tableName = "test1"; hiveSyncConfig.basePath = "/tmp/hdfs/HiveSyncToolTest/"; hiveSyncConfig.assumeDatePartitioning = true; + hiveSyncConfig.usePreApacheInputFormat = false; hiveSyncConfig.partitionFields = Lists.newArrayList("datestr"); dtfOut = DateTimeFormat.forPattern("yyyy/MM/dd"); @@ -309,7 +311,7 @@ public class TestUtil { .overBaseCommit(dataFile.getCommitTime()).withFs(fileSystem).build(); List records = (isLogSchemaSimple ? SchemaTestUtil.generateTestRecords(0, 100) : SchemaTestUtil.generateEvolvedTestRecords(100, 100)); - Map header = Maps.newHashMap(); + Map header = Maps.newHashMap(); header.put(HoodieLogBlock.HeaderMetadataType.INSTANT_TIME, dataFile.getCommitTime()); header.put(HoodieLogBlock.HeaderMetadataType.SCHEMA, schema.toString()); HoodieAvroDataBlock dataBlock = new HoodieAvroDataBlock(records, header); diff --git a/hoodie-hive/src/test/java/com/uber/hoodie/hive/util/HiveTestService.java b/hudi-hive/src/test/java/org/apache/hudi/hive/util/HiveTestService.java similarity index 99% rename from hoodie-hive/src/test/java/com/uber/hoodie/hive/util/HiveTestService.java rename to hudi-hive/src/test/java/org/apache/hudi/hive/util/HiveTestService.java index 2a7451f26..74e8d79d0 100644 --- a/hoodie-hive/src/test/java/com/uber/hoodie/hive/util/HiveTestService.java +++ b/hudi-hive/src/test/java/org/apache/hudi/hive/util/HiveTestService.java @@ -16,12 +16,11 @@ * limitations under the License. */ -package com.uber.hoodie.hive.util; +package org.apache.hudi.hive.util; import com.google.common.base.Preconditions; import com.google.common.collect.Maps; import com.google.common.io.Files; -import com.uber.hoodie.common.model.HoodieTestUtils; import java.io.File; import java.io.IOException; import java.net.InetSocketAddress; @@ -41,6 +40,7 @@ import org.apache.hadoop.hive.metastore.TUGIBasedProcessor; import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.thrift.TUGIContainingTransport; import org.apache.hive.service.server.HiveServer2; +import org.apache.hudi.common.model.HoodieTestUtils; import org.apache.thrift.TProcessor; import org.apache.thrift.protocol.TBinaryProtocol; import org.apache.thrift.server.TServer; diff --git a/hoodie-hive/src/test/resources/log4j-surefire-quiet.properties b/hudi-hive/src/test/resources/log4j-surefire-quiet.properties similarity index 100% rename from hoodie-hive/src/test/resources/log4j-surefire-quiet.properties rename to hudi-hive/src/test/resources/log4j-surefire-quiet.properties diff --git a/hoodie-hive/src/test/resources/log4j-surefire.properties b/hudi-hive/src/test/resources/log4j-surefire.properties similarity index 100% rename from hoodie-hive/src/test/resources/log4j-surefire.properties rename to hudi-hive/src/test/resources/log4j-surefire.properties diff --git a/hoodie-integ-test/pom.xml b/hudi-integ-test/pom.xml similarity index 92% rename from hoodie-integ-test/pom.xml rename to hudi-integ-test/pom.xml index 07a41544f..6b7e29e78 100644 --- a/hoodie-integ-test/pom.xml +++ b/hudi-integ-test/pom.xml @@ -1,12 +1,12 @@ - hoodie - com.uber.hoodie - 0.4.8-SNAPSHOT + hudi + org.apache.hudi + 0.5.0-SNAPSHOT ../pom.xml - hoodie-integ-test + hudi-integ-test 4.0.0 @@ -25,8 +25,8 @@ 2.17 - com.uber.hoodie - hoodie-spark + org.apache.hudi + hudi-spark ${project.version} @@ -36,8 +36,8 @@ - com.uber.hoodie - hoodie-common + org.apache.hudi + hudi-common ${project.version} tests test-jar @@ -50,8 +50,8 @@ test - com.uber.hoodie - hoodie-spark + org.apache.hudi + hudi-spark ${project.version} tests test-jar @@ -100,8 +100,8 @@ - com.uber.hoodie - hoodie-hadoop-sparkworker-docker + org.apache.hudi + hudi-hadoop-sparkworker-docker ${project.version} pom import diff --git a/hoodie-integ-test/src/main/resources/META-INF/LICENSE.txt b/hudi-integ-test/src/main/resources/META-INF/LICENSE.txt similarity index 100% rename from hoodie-integ-test/src/main/resources/META-INF/LICENSE.txt rename to hudi-integ-test/src/main/resources/META-INF/LICENSE.txt diff --git a/hoodie-integ-test/src/main/resources/META-INF/NOTICE.txt b/hudi-integ-test/src/main/resources/META-INF/NOTICE.txt similarity index 100% rename from hoodie-integ-test/src/main/resources/META-INF/NOTICE.txt rename to hudi-integ-test/src/main/resources/META-INF/NOTICE.txt diff --git a/hoodie-integ-test/src/test/java/com/uber/hoodie/integ/ITTestBase.java b/hudi-integ-test/src/test/java/org/apache/hudi/integ/ITTestBase.java similarity index 99% rename from hoodie-integ-test/src/test/java/com/uber/hoodie/integ/ITTestBase.java rename to hudi-integ-test/src/test/java/org/apache/hudi/integ/ITTestBase.java index 995afe2a4..9751c11ff 100644 --- a/hoodie-integ-test/src/test/java/com/uber/hoodie/integ/ITTestBase.java +++ b/hudi-integ-test/src/test/java/org/apache/hudi/integ/ITTestBase.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package com.uber.hoodie.integ; +package org.apache.hudi.integ; import static java.util.concurrent.TimeUnit.SECONDS; import static org.awaitility.Awaitility.await; diff --git a/hoodie-integ-test/src/test/java/com/uber/hoodie/integ/ITTestHoodieSanity.java b/hudi-integ-test/src/test/java/org/apache/hudi/integ/ITTestHoodieSanity.java similarity index 99% rename from hoodie-integ-test/src/test/java/com/uber/hoodie/integ/ITTestHoodieSanity.java rename to hudi-integ-test/src/test/java/org/apache/hudi/integ/ITTestHoodieSanity.java index d5d5220d2..89eb56960 100644 --- a/hoodie-integ-test/src/test/java/com/uber/hoodie/integ/ITTestHoodieSanity.java +++ b/hudi-integ-test/src/test/java/org/apache/hudi/integ/ITTestHoodieSanity.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package com.uber.hoodie.integ; +package org.apache.hudi.integ; import java.util.Arrays; import org.junit.Assert; diff --git a/hoodie-integ-test/src/test/resources/log4j-surefire-quiet.properties b/hudi-integ-test/src/test/resources/log4j-surefire-quiet.properties similarity index 100% rename from hoodie-integ-test/src/test/resources/log4j-surefire-quiet.properties rename to hudi-integ-test/src/test/resources/log4j-surefire-quiet.properties diff --git a/hoodie-integ-test/src/test/resources/log4j-surefire.properties b/hudi-integ-test/src/test/resources/log4j-surefire.properties similarity index 100% rename from hoodie-integ-test/src/test/resources/log4j-surefire.properties rename to hudi-integ-test/src/test/resources/log4j-surefire.properties diff --git a/hoodie-spark/pom.xml b/hudi-spark/pom.xml similarity index 92% rename from hoodie-spark/pom.xml rename to hudi-spark/pom.xml index 201f68a00..2ca54201d 100644 --- a/hoodie-spark/pom.xml +++ b/hudi-spark/pom.xml @@ -17,14 +17,14 @@ --> - hoodie - com.uber.hoodie - 0.4.8-SNAPSHOT + hudi + org.apache.hudi + 0.5.0-SNAPSHOT 4.0.0 - com.uber.hoodie - hoodie-spark + org.apache.hudi + hudi-spark jar @@ -246,23 +246,23 @@ - com.uber.hoodie - hoodie-client + org.apache.hudi + hudi-client ${project.version} - com.uber.hoodie - hoodie-common + org.apache.hudi + hudi-common ${project.version} - com.uber.hoodie - hoodie-hadoop-mr + org.apache.hudi + hudi-hadoop-mr ${project.version} - com.uber.hoodie - hoodie-hive + org.apache.hudi + hudi-hive ${project.version} @@ -272,16 +272,16 @@ test - com.uber.hoodie - hoodie-client + org.apache.hudi + hudi-client ${project.version} tests test-jar test - com.uber.hoodie - hoodie-common + org.apache.hudi + hudi-common ${project.version} tests test-jar diff --git a/hoodie-spark/run_hoodie_app.sh b/hudi-spark/run_hoodie_app.sh similarity index 64% rename from hoodie-spark/run_hoodie_app.sh rename to hudi-spark/run_hoodie_app.sh index c346ba4a9..8c2e54b05 100755 --- a/hoodie-spark/run_hoodie_app.sh +++ b/hudi-spark/run_hoodie_app.sh @@ -7,7 +7,7 @@ function error_exit { DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" #Ensure we pick the right jar even for hive11 builds -HOODIE_JAR=`ls -c $DIR/../packaging/hoodie-spark-bundle/target/hoodie-spark-bundle-*.jar | grep -v source | head -1` +HUDI_JAR=`ls -c $DIR/../packaging/hudi-spark-bundle/target/hudi-spark-bundle-*.jar | grep -v source | head -1` if [ -z "$HADOOP_CONF_DIR" ]; then echo "setting hadoop conf dir" @@ -20,5 +20,5 @@ fi OTHER_JARS=`ls -1 $DIR/target/lib/*jar | grep -v '*avro*-1.' | tr '\n' ':'` #TODO - Need to move TestDataGenerator and HoodieJavaApp out of tests -echo "Running command : java -cp $DIR/target/test-classes/:$DIR/../hoodie-client/target/test-classes/:${HADOOP_CONF_DIR}:$HOODIE_JAR:${CLIENT_JAR}:$OTHER_JARS HoodieJavaApp $@" -java -Xmx1G -cp $DIR/target/test-classes/:$DIR/../hoodie-client/target/test-classes/:${HADOOP_CONF_DIR}:$HOODIE_JAR:${CLIENT_JAR}:$OTHER_JARS HoodieJavaApp "$@" +echo "Running command : java -cp $DIR/target/test-classes/:$DIR/../hudi-client/target/test-classes/:${HADOOP_CONF_DIR}:$HUDI_JAR:${CLIENT_JAR}:$OTHER_JARS HoodieJavaApp $@" +java -Xmx1G -cp $DIR/target/test-classes/:$DIR/../hudi-client/target/test-classes/:${HADOOP_CONF_DIR}:$HUDI_JAR:${CLIENT_JAR}:$OTHER_JARS HoodieJavaApp "$@" diff --git a/hoodie-spark/src/main/java/com/uber/hoodie/BaseAvroPayload.java b/hudi-spark/src/main/java/org/apache/hudi/BaseAvroPayload.java similarity index 90% rename from hoodie-spark/src/main/java/com/uber/hoodie/BaseAvroPayload.java rename to hudi-spark/src/main/java/org/apache/hudi/BaseAvroPayload.java index 55f52d176..38acdea3b 100644 --- a/hoodie-spark/src/main/java/com/uber/hoodie/BaseAvroPayload.java +++ b/hudi-spark/src/main/java/org/apache/hudi/BaseAvroPayload.java @@ -16,14 +16,14 @@ * limitations under the License. */ -package com.uber.hoodie; +package org.apache.hudi; -import com.uber.hoodie.common.util.HoodieAvroUtils; -import com.uber.hoodie.exception.HoodieException; -import com.uber.hoodie.exception.HoodieIOException; import java.io.IOException; import java.io.Serializable; import org.apache.avro.generic.GenericRecord; +import org.apache.hudi.common.util.HoodieAvroUtils; +import org.apache.hudi.exception.HoodieException; +import org.apache.hudi.exception.HoodieIOException; /** * Base class for all AVRO record based payloads, that can be ordered based on a field diff --git a/hoodie-spark/src/main/java/com/uber/hoodie/ComplexKeyGenerator.java b/hudi-spark/src/main/java/org/apache/hudi/ComplexKeyGenerator.java similarity index 94% rename from hoodie-spark/src/main/java/com/uber/hoodie/ComplexKeyGenerator.java rename to hudi-spark/src/main/java/org/apache/hudi/ComplexKeyGenerator.java index 69d80f7ad..cc2ccb277 100644 --- a/hoodie-spark/src/main/java/com/uber/hoodie/ComplexKeyGenerator.java +++ b/hudi-spark/src/main/java/org/apache/hudi/ComplexKeyGenerator.java @@ -16,14 +16,14 @@ * limitations under the License. */ -package com.uber.hoodie; +package org.apache.hudi; -import com.uber.hoodie.common.model.HoodieKey; -import com.uber.hoodie.common.util.TypedProperties; -import com.uber.hoodie.exception.HoodieException; import java.util.Arrays; import java.util.List; import org.apache.avro.generic.GenericRecord; +import org.apache.hudi.common.model.HoodieKey; +import org.apache.hudi.common.util.TypedProperties; +import org.apache.hudi.exception.HoodieException; /** * Complex key generator, which takes names of fields to be used for recordKey and partitionPath as @@ -78,4 +78,4 @@ public class ComplexKeyGenerator extends KeyGenerator { public List getPartitionPathFields() { return partitionPathFields; } -} \ No newline at end of file +} diff --git a/hoodie-spark/src/main/java/com/uber/hoodie/DataSourceUtils.java b/hudi-spark/src/main/java/org/apache/hudi/DataSourceUtils.java similarity index 89% rename from hoodie-spark/src/main/java/com/uber/hoodie/DataSourceUtils.java rename to hudi-spark/src/main/java/org/apache/hudi/DataSourceUtils.java index ad754d684..3c9709855 100644 --- a/hoodie-spark/src/main/java/com/uber/hoodie/DataSourceUtils.java +++ b/hudi-spark/src/main/java/org/apache/hudi/DataSourceUtils.java @@ -16,25 +16,8 @@ * limitations under the License. */ -package com.uber.hoodie; +package org.apache.hudi; -import com.uber.hoodie.client.embedded.EmbeddedTimelineService; -import com.uber.hoodie.common.model.HoodieKey; -import com.uber.hoodie.common.model.HoodieRecord; -import com.uber.hoodie.common.model.HoodieRecordPayload; -import com.uber.hoodie.common.util.Option; -import com.uber.hoodie.common.util.ReflectionUtils; -import com.uber.hoodie.common.util.TypedProperties; -import com.uber.hoodie.config.HoodieCompactionConfig; -import com.uber.hoodie.config.HoodieIndexConfig; -import com.uber.hoodie.config.HoodieWriteConfig; -import com.uber.hoodie.exception.DatasetNotFoundException; -import com.uber.hoodie.exception.HoodieException; -import com.uber.hoodie.exception.HoodieNotSupportedException; -import com.uber.hoodie.hive.HiveSyncConfig; -import com.uber.hoodie.hive.PartitionValueExtractor; -import com.uber.hoodie.hive.SlashEncodedDayPartitionValueExtractor; -import com.uber.hoodie.index.HoodieIndex; import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; @@ -43,6 +26,23 @@ import java.util.Map; import java.util.stream.Collectors; import org.apache.avro.Schema.Field; import org.apache.avro.generic.GenericRecord; +import org.apache.hudi.client.embedded.EmbeddedTimelineService; +import org.apache.hudi.common.model.HoodieKey; +import org.apache.hudi.common.model.HoodieRecord; +import org.apache.hudi.common.model.HoodieRecordPayload; +import org.apache.hudi.common.util.Option; +import org.apache.hudi.common.util.ReflectionUtils; +import org.apache.hudi.common.util.TypedProperties; +import org.apache.hudi.config.HoodieCompactionConfig; +import org.apache.hudi.config.HoodieIndexConfig; +import org.apache.hudi.config.HoodieWriteConfig; +import org.apache.hudi.exception.DatasetNotFoundException; +import org.apache.hudi.exception.HoodieException; +import org.apache.hudi.exception.HoodieNotSupportedException; +import org.apache.hudi.hive.HiveSyncConfig; +import org.apache.hudi.hive.PartitionValueExtractor; +import org.apache.hudi.hive.SlashEncodedDayPartitionValueExtractor; +import org.apache.hudi.index.HoodieIndex; import org.apache.spark.api.java.JavaRDD; import org.apache.spark.api.java.JavaSparkContext; @@ -225,6 +225,9 @@ public class DataSourceUtils { checkRequiredProperties(props, Arrays.asList(DataSourceWriteOptions.HIVE_TABLE_OPT_KEY())); HiveSyncConfig hiveSyncConfig = new HiveSyncConfig(); hiveSyncConfig.basePath = basePath; + hiveSyncConfig.usePreApacheInputFormat = + props.getBoolean(DataSourceWriteOptions.HIVE_USE_PRE_APACHE_INPUT_FORMAT_OPT_KEY(), + Boolean.valueOf(DataSourceWriteOptions.DEFAULT_USE_PRE_APACHE_INPUT_FORMAT_OPT_VAL())); hiveSyncConfig.assumeDatePartitioning = props.getBoolean(DataSourceWriteOptions.HIVE_ASSUME_DATE_PARTITION_OPT_KEY(), Boolean.valueOf(DataSourceWriteOptions.DEFAULT_HIVE_ASSUME_DATE_PARTITION_OPT_VAL())); diff --git a/hoodie-spark/src/main/java/com/uber/hoodie/EmptyHoodieRecordPayload.java b/hudi-spark/src/main/java/org/apache/hudi/EmptyHoodieRecordPayload.java similarity index 92% rename from hoodie-spark/src/main/java/com/uber/hoodie/EmptyHoodieRecordPayload.java rename to hudi-spark/src/main/java/org/apache/hudi/EmptyHoodieRecordPayload.java index 67be81b96..0d8982dcb 100644 --- a/hoodie-spark/src/main/java/com/uber/hoodie/EmptyHoodieRecordPayload.java +++ b/hudi-spark/src/main/java/org/apache/hudi/EmptyHoodieRecordPayload.java @@ -16,13 +16,13 @@ * limitations under the License. */ -package com.uber.hoodie; +package org.apache.hudi; -import com.uber.hoodie.common.model.HoodieRecordPayload; -import com.uber.hoodie.common.util.Option; import org.apache.avro.Schema; import org.apache.avro.generic.GenericRecord; import org.apache.avro.generic.IndexedRecord; +import org.apache.hudi.common.model.HoodieRecordPayload; +import org.apache.hudi.common.util.Option; /** * Empty payload used for deletions diff --git a/hoodie-spark/src/main/java/com/uber/hoodie/HoodieDataSourceHelpers.java b/hudi-spark/src/main/java/org/apache/hudi/HoodieDataSourceHelpers.java similarity index 90% rename from hoodie-spark/src/main/java/com/uber/hoodie/HoodieDataSourceHelpers.java rename to hudi-spark/src/main/java/org/apache/hudi/HoodieDataSourceHelpers.java index 8bbaf5aff..a5573fb62 100644 --- a/hoodie-spark/src/main/java/com/uber/hoodie/HoodieDataSourceHelpers.java +++ b/hudi-spark/src/main/java/org/apache/hudi/HoodieDataSourceHelpers.java @@ -16,17 +16,17 @@ * limitations under the License. */ -package com.uber.hoodie; +package org.apache.hudi; import com.google.common.collect.Sets; -import com.uber.hoodie.common.model.HoodieTableType; -import com.uber.hoodie.common.table.HoodieTableMetaClient; -import com.uber.hoodie.common.table.HoodieTimeline; -import com.uber.hoodie.common.table.timeline.HoodieActiveTimeline; -import com.uber.hoodie.common.table.timeline.HoodieInstant; import java.util.List; import java.util.stream.Collectors; import org.apache.hadoop.fs.FileSystem; +import org.apache.hudi.common.model.HoodieTableType; +import org.apache.hudi.common.table.HoodieTableMetaClient; +import org.apache.hudi.common.table.HoodieTimeline; +import org.apache.hudi.common.table.timeline.HoodieActiveTimeline; +import org.apache.hudi.common.table.timeline.HoodieInstant; /** * List of helpers to aid, construction of instanttime for read and write operations using diff --git a/hoodie-spark/src/main/java/com/uber/hoodie/KeyGenerator.java b/hudi-spark/src/main/java/org/apache/hudi/KeyGenerator.java similarity index 88% rename from hoodie-spark/src/main/java/com/uber/hoodie/KeyGenerator.java rename to hudi-spark/src/main/java/org/apache/hudi/KeyGenerator.java index 40bb22d7d..583db3c10 100644 --- a/hoodie-spark/src/main/java/com/uber/hoodie/KeyGenerator.java +++ b/hudi-spark/src/main/java/org/apache/hudi/KeyGenerator.java @@ -16,16 +16,16 @@ * limitations under the License. */ -package com.uber.hoodie; +package org.apache.hudi; -import com.uber.hoodie.common.model.HoodieKey; -import com.uber.hoodie.common.util.TypedProperties; import java.io.Serializable; import org.apache.avro.generic.GenericRecord; +import org.apache.hudi.common.model.HoodieKey; +import org.apache.hudi.common.util.TypedProperties; /** * Abstract class to extend for plugging in extraction of - * {@link com.uber.hoodie.common.model.HoodieKey} + * {@link HoodieKey} * from an Avro record */ public abstract class KeyGenerator implements Serializable { diff --git a/hoodie-spark/src/main/java/com/uber/hoodie/NonpartitionedKeyGenerator.java b/hudi-spark/src/main/java/org/apache/hudi/NonpartitionedKeyGenerator.java similarity index 91% rename from hoodie-spark/src/main/java/com/uber/hoodie/NonpartitionedKeyGenerator.java rename to hudi-spark/src/main/java/org/apache/hudi/NonpartitionedKeyGenerator.java index 031e8cdc9..50cf327d4 100644 --- a/hoodie-spark/src/main/java/com/uber/hoodie/NonpartitionedKeyGenerator.java +++ b/hudi-spark/src/main/java/org/apache/hudi/NonpartitionedKeyGenerator.java @@ -16,11 +16,11 @@ * limitations under the License. */ -package com.uber.hoodie; +package org.apache.hudi; -import com.uber.hoodie.common.model.HoodieKey; -import com.uber.hoodie.common.util.TypedProperties; import org.apache.avro.generic.GenericRecord; +import org.apache.hudi.common.model.HoodieKey; +import org.apache.hudi.common.util.TypedProperties; /** * Simple Key generator for unpartitioned Hive Tables diff --git a/hoodie-spark/src/main/java/com/uber/hoodie/OverwriteWithLatestAvroPayload.java b/hudi-spark/src/main/java/org/apache/hudi/OverwriteWithLatestAvroPayload.java similarity index 93% rename from hoodie-spark/src/main/java/com/uber/hoodie/OverwriteWithLatestAvroPayload.java rename to hudi-spark/src/main/java/org/apache/hudi/OverwriteWithLatestAvroPayload.java index 5a9a5e43f..39e2df706 100644 --- a/hoodie-spark/src/main/java/com/uber/hoodie/OverwriteWithLatestAvroPayload.java +++ b/hudi-spark/src/main/java/org/apache/hudi/OverwriteWithLatestAvroPayload.java @@ -16,15 +16,15 @@ * limitations under the License. */ -package com.uber.hoodie; +package org.apache.hudi; -import com.uber.hoodie.common.model.HoodieRecordPayload; -import com.uber.hoodie.common.util.HoodieAvroUtils; -import com.uber.hoodie.common.util.Option; import java.io.IOException; import org.apache.avro.Schema; import org.apache.avro.generic.GenericRecord; import org.apache.avro.generic.IndexedRecord; +import org.apache.hudi.common.model.HoodieRecordPayload; +import org.apache.hudi.common.util.HoodieAvroUtils; +import org.apache.hudi.common.util.Option; /** * Default payload used for delta streamer. diff --git a/hoodie-spark/src/main/java/com/uber/hoodie/SimpleKeyGenerator.java b/hudi-spark/src/main/java/org/apache/hudi/SimpleKeyGenerator.java similarity index 92% rename from hoodie-spark/src/main/java/com/uber/hoodie/SimpleKeyGenerator.java rename to hudi-spark/src/main/java/org/apache/hudi/SimpleKeyGenerator.java index 31b2787e9..fec48ffe4 100644 --- a/hoodie-spark/src/main/java/com/uber/hoodie/SimpleKeyGenerator.java +++ b/hudi-spark/src/main/java/org/apache/hudi/SimpleKeyGenerator.java @@ -16,12 +16,12 @@ * limitations under the License. */ -package com.uber.hoodie; +package org.apache.hudi; -import com.uber.hoodie.common.model.HoodieKey; -import com.uber.hoodie.common.util.TypedProperties; -import com.uber.hoodie.exception.HoodieException; import org.apache.avro.generic.GenericRecord; +import org.apache.hudi.common.model.HoodieKey; +import org.apache.hudi.common.util.TypedProperties; +import org.apache.hudi.exception.HoodieException; /** * Simple key generator, which takes names of fields to be used for recordKey and partitionPath as diff --git a/hoodie-spark/src/main/resources/META-INF/LICENSE.txt b/hudi-spark/src/main/resources/META-INF/LICENSE.txt similarity index 100% rename from hoodie-spark/src/main/resources/META-INF/LICENSE.txt rename to hudi-spark/src/main/resources/META-INF/LICENSE.txt diff --git a/hoodie-spark/src/main/resources/META-INF/NOTICE.txt b/hudi-spark/src/main/resources/META-INF/NOTICE.txt similarity index 100% rename from hoodie-spark/src/main/resources/META-INF/NOTICE.txt rename to hudi-spark/src/main/resources/META-INF/NOTICE.txt diff --git a/hoodie-spark/src/main/scala/com/uber/hoodie/AvroConversionUtils.scala b/hudi-spark/src/main/scala/org/apache/hudi/AvroConversionUtils.scala similarity index 99% rename from hoodie-spark/src/main/scala/com/uber/hoodie/AvroConversionUtils.scala rename to hudi-spark/src/main/scala/org/apache/hudi/AvroConversionUtils.scala index 5671fe919..e119979a1 100644 --- a/hoodie-spark/src/main/scala/com/uber/hoodie/AvroConversionUtils.scala +++ b/hudi-spark/src/main/scala/org/apache/hudi/AvroConversionUtils.scala @@ -15,7 +15,7 @@ * limitations under the License. */ -package com.uber.hoodie +package org.apache.hudi import java.nio.ByteBuffer import java.sql.{Date, Timestamp} diff --git a/hoodie-spark/src/main/scala/com/uber/hoodie/DataSourceOptions.scala b/hudi-spark/src/main/scala/org/apache/hudi/DataSourceOptions.scala similarity index 96% rename from hoodie-spark/src/main/scala/com/uber/hoodie/DataSourceOptions.scala rename to hudi-spark/src/main/scala/org/apache/hudi/DataSourceOptions.scala index b7f534c4c..bbcc98c87 100644 --- a/hoodie-spark/src/main/scala/com/uber/hoodie/DataSourceOptions.scala +++ b/hudi-spark/src/main/scala/org/apache/hudi/DataSourceOptions.scala @@ -15,10 +15,10 @@ * limitations under the License. */ -package com.uber.hoodie +package org.apache.hudi -import com.uber.hoodie.common.model.HoodieTableType -import com.uber.hoodie.hive.SlashEncodedDayPartitionValueExtractor +import org.apache.hudi.common.model.HoodieTableType +import org.apache.hudi.hive.SlashEncodedDayPartitionValueExtractor /** * List of options that can be passed to the Hoodie datasource, @@ -192,6 +192,7 @@ object DataSourceWriteOptions { val HIVE_PARTITION_FIELDS_OPT_KEY = "hoodie.datasource.hive_sync.partition_fields" val HIVE_PARTITION_EXTRACTOR_CLASS_OPT_KEY = "hoodie.datasource.hive_sync.partition_extractor_class" val HIVE_ASSUME_DATE_PARTITION_OPT_KEY = "hoodie.datasource.hive_sync.assume_date_partitioning" + val HIVE_USE_PRE_APACHE_INPUT_FORMAT_OPT_KEY = "hoodie.datasource.hive_sync.use_pre_apache_input_format" // DEFAULT FOR HIVE SPECIFIC CONFIGS val DEFAULT_HIVE_SYNC_ENABLED_OPT_VAL = "false" @@ -203,4 +204,5 @@ object DataSourceWriteOptions { val DEFAULT_HIVE_PARTITION_FIELDS_OPT_VAL = "" val DEFAULT_HIVE_PARTITION_EXTRACTOR_CLASS_OPT_VAL = classOf[SlashEncodedDayPartitionValueExtractor].getCanonicalName val DEFAULT_HIVE_ASSUME_DATE_PARTITION_OPT_VAL = "false" + val DEFAULT_USE_PRE_APACHE_INPUT_FORMAT_OPT_VAL = "false" } diff --git a/hoodie-spark/src/main/scala/com/uber/hoodie/DefaultSource.scala b/hudi-spark/src/main/scala/org/apache/hudi/DefaultSource.scala similarity index 93% rename from hoodie-spark/src/main/scala/com/uber/hoodie/DefaultSource.scala rename to hudi-spark/src/main/scala/org/apache/hudi/DefaultSource.scala index 71f3485de..18f3dba8a 100644 --- a/hoodie-spark/src/main/scala/com/uber/hoodie/DefaultSource.scala +++ b/hudi-spark/src/main/scala/org/apache/hudi/DefaultSource.scala @@ -15,17 +15,18 @@ * limitations under the License. */ -package com.uber.hoodie +package org.apache.hudi -import com.uber.hoodie.DataSourceReadOptions._ -import com.uber.hoodie.exception.HoodieException +import org.apache.hudi.DataSourceReadOptions._ +import org.apache.hudi.exception.HoodieException +import org.apache.hudi.hadoop.HoodieROTablePathFilter import org.apache.log4j.LogManager -import org.apache.spark.sql.{DataFrame, SaveMode, SQLContext} import org.apache.spark.sql.execution.datasources.DataSource import org.apache.spark.sql.execution.streaming.Sink import org.apache.spark.sql.sources._ import org.apache.spark.sql.streaming.OutputMode import org.apache.spark.sql.types.StructType +import org.apache.spark.sql.{DataFrame, SQLContext, SaveMode} /** * Hoodie Spark Datasource, for reading and writing hoodie datasets @@ -67,7 +68,7 @@ class DefaultSource extends RelationProvider // non-hoodie/hoodie path files. set the path filter up sqlContext.sparkContext.hadoopConfiguration.setClass( "mapreduce.input.pathFilter.class", - classOf[com.uber.hoodie.hadoop.HoodieROTablePathFilter], + classOf[HoodieROTablePathFilter], classOf[org.apache.hadoop.fs.PathFilter]); log.info("Constructing hoodie (as parquet) data source with options :" + parameters) diff --git a/hoodie-spark/src/main/scala/com/uber/hoodie/HoodieSparkSqlWriter.scala b/hudi-spark/src/main/scala/org/apache/hudi/HoodieSparkSqlWriter.scala similarity index 92% rename from hoodie-spark/src/main/scala/com/uber/hoodie/HoodieSparkSqlWriter.scala rename to hudi-spark/src/main/scala/org/apache/hudi/HoodieSparkSqlWriter.scala index 32bb72da1..ad67364ab 100644 --- a/hoodie-spark/src/main/scala/com/uber/hoodie/HoodieSparkSqlWriter.scala +++ b/hudi-spark/src/main/scala/org/apache/hudi/HoodieSparkSqlWriter.scala @@ -15,20 +15,19 @@ * limitations under the License. */ -package com.uber.hoodie +package org.apache.hudi import java.util -import com.uber.hoodie.DataSourceWriteOptions._ -import com.uber.hoodie.common.table.HoodieTableMetaClient -import com.uber.hoodie.common.util.{FSUtils, TypedProperties} -import com.uber.hoodie.config.HoodieWriteConfig -import com.uber.hoodie.exception.HoodieException -import com.uber.hoodie.hive.{HiveSyncConfig, HiveSyncTool} -import com.uber.hoodie.common.util.Option import org.apache.avro.generic.GenericRecord import org.apache.hadoop.fs.{FileSystem, Path} import org.apache.hadoop.hive.conf.HiveConf +import org.apache.hudi.DataSourceWriteOptions._ +import org.apache.hudi.common.table.HoodieTableMetaClient +import org.apache.hudi.common.util.{FSUtils, TypedProperties} +import org.apache.hudi.config.HoodieWriteConfig +import org.apache.hudi.exception.HoodieException +import org.apache.hudi.hive.{HiveSyncConfig, HiveSyncTool} import org.apache.log4j.LogManager import org.apache.spark.api.java.JavaSparkContext import org.apache.spark.rdd.RDD @@ -37,14 +36,14 @@ import org.apache.spark.sql.{DataFrame, SQLContext, SaveMode} import scala.collection.JavaConversions._ import scala.collection.mutable.ListBuffer -private[hoodie] object HoodieSparkSqlWriter { +private[hudi] object HoodieSparkSqlWriter { private val log = LogManager.getLogger("HoodieSparkSQLWriter") def write(sqlContext: SQLContext, mode: SaveMode, parameters: Map[String, String], - df: DataFrame): (Boolean, Option[String]) = { + df: DataFrame): (Boolean, common.util.Option[String]) = { val sparkContext = sqlContext.sparkContext val path = parameters.get("path") @@ -105,7 +104,7 @@ private[hoodie] object HoodieSparkSqlWriter { } if (mode == SaveMode.Ignore && exists) { log.warn(s"hoodie dataset at $basePath already exists. Ignoring & not performing actual writes.") - return (true, Option.empty()) + return (true, common.util.Option.empty()) } if (mode == SaveMode.Overwrite && exists) { log.warn(s"hoodie dataset at $basePath already exists. Deleting existing data & overwriting with new data.") @@ -136,7 +135,7 @@ private[hoodie] object HoodieSparkSqlWriter { if (hoodieRecords.isEmpty()) { log.info("new batch has no new records, skipping...") - return (true, Option.empty()) + return (true, common.util.Option.empty()) } val commitTime = client.startCommit() @@ -153,7 +152,7 @@ private[hoodie] object HoodieSparkSqlWriter { client.commit(commitTime, writeStatuses) } else { client.commit(commitTime, writeStatuses, - Option.of(new util.HashMap[String, String](mapAsJavaMap(metaMap)))) + common.util.Option.of(new util.HashMap[String, String](mapAsJavaMap(metaMap)))) } if (commitSuccess) { @@ -189,7 +188,7 @@ private[hoodie] object HoodieSparkSqlWriter { } false } - (writeSuccessful, Option.ofNullable(commitTime)) + (writeSuccessful, common.util.Option.ofNullable(commitTime)) } /** @@ -240,6 +239,8 @@ private[hoodie] object HoodieSparkSqlWriter { private def buildSyncConfig(basePath: Path, parameters: Map[String, String]): HiveSyncConfig = { val hiveSyncConfig: HiveSyncConfig = new HiveSyncConfig() hiveSyncConfig.basePath = basePath.toString + hiveSyncConfig.usePreApacheInputFormat = + parameters.get(HIVE_USE_PRE_APACHE_INPUT_FORMAT_OPT_KEY).exists(r => r.toBoolean) hiveSyncConfig.assumeDatePartitioning = parameters.get(HIVE_ASSUME_DATE_PARTITION_OPT_KEY).exists(r => r.toBoolean) hiveSyncConfig.databaseName = parameters(HIVE_DATABASE_OPT_KEY) diff --git a/hoodie-spark/src/main/scala/com/uber/hoodie/HoodieStreamingSink.scala b/hudi-spark/src/main/scala/org/apache/hudi/HoodieStreamingSink.scala similarity index 98% rename from hoodie-spark/src/main/scala/com/uber/hoodie/HoodieStreamingSink.scala rename to hudi-spark/src/main/scala/org/apache/hudi/HoodieStreamingSink.scala index 2d0776f93..9f18a2eb3 100644 --- a/hoodie-spark/src/main/scala/com/uber/hoodie/HoodieStreamingSink.scala +++ b/hudi-spark/src/main/scala/org/apache/hudi/HoodieStreamingSink.scala @@ -14,13 +14,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.uber.hoodie +package org.apache.hudi -import com.uber.hoodie.exception.HoodieCorruptedDataException -import org.apache.spark.sql.{DataFrame, SQLContext, SaveMode} +import org.apache.hudi.exception.HoodieCorruptedDataException +import org.apache.log4j.LogManager import org.apache.spark.sql.execution.streaming.Sink import org.apache.spark.sql.streaming.OutputMode -import org.apache.log4j.LogManager +import org.apache.spark.sql.{DataFrame, SQLContext, SaveMode} import scala.util.{Failure, Success, Try} @@ -119,4 +119,4 @@ class HoodieStreamingSink(sqlContext: SQLContext, case f => f } } -} \ No newline at end of file +} diff --git a/hoodie-spark/src/main/scala/com/uber/hoodie/IncrementalRelation.scala b/hudi-spark/src/main/scala/org/apache/hudi/IncrementalRelation.scala similarity index 94% rename from hoodie-spark/src/main/scala/com/uber/hoodie/IncrementalRelation.scala rename to hudi-spark/src/main/scala/org/apache/hudi/IncrementalRelation.scala index 5d89e1b1d..91b890943 100644 --- a/hoodie-spark/src/main/scala/com/uber/hoodie/IncrementalRelation.scala +++ b/hudi-spark/src/main/scala/org/apache/hudi/IncrementalRelation.scala @@ -15,15 +15,15 @@ * limitations under the License. */ -package com.uber.hoodie +package org.apache.hudi -import com.uber.hoodie.common.model.{HoodieCommitMetadata, HoodieRecord, HoodieTableType} -import com.uber.hoodie.common.table.HoodieTableMetaClient -import com.uber.hoodie.common.util.ParquetUtils -import com.uber.hoodie.config.HoodieWriteConfig -import com.uber.hoodie.exception.HoodieException -import com.uber.hoodie.table.HoodieTable import org.apache.hadoop.fs.Path +import org.apache.hudi.common.model.{HoodieCommitMetadata, HoodieRecord, HoodieTableType} +import org.apache.hudi.common.table.HoodieTableMetaClient +import org.apache.hudi.common.util.ParquetUtils +import org.apache.hudi.config.HoodieWriteConfig +import org.apache.hudi.exception.HoodieException +import org.apache.hudi.table.HoodieTable import org.apache.log4j.LogManager import org.apache.spark.rdd.RDD import org.apache.spark.sql.sources.{BaseRelation, TableScan} diff --git a/hoodie-spark/src/main/scala/com/uber/hoodie/package.scala b/hudi-spark/src/main/scala/org/apache/hudi/package.scala similarity index 86% rename from hoodie-spark/src/main/scala/com/uber/hoodie/package.scala rename to hudi-spark/src/main/scala/org/apache/hudi/package.scala index dd161c04e..4d6025fa5 100644 --- a/hoodie-spark/src/main/scala/com/uber/hoodie/package.scala +++ b/hudi-spark/src/main/scala/org/apache/hudi/package.scala @@ -15,24 +15,24 @@ * limitations under the License. */ -package com.uber.hoodie +package org.apache import org.apache.spark.sql.{DataFrame, DataFrameReader, DataFrameWriter} -package object hoodie { +package object hudi { /** * Adds a method, `hoodie`, to DataFrameWriter */ implicit class AvroDataFrameWriter[T](writer: DataFrameWriter[T]) { - def avro: String => Unit = writer.format("com.uber.hoodie").save + def avro: String => Unit = writer.format("org.apache.hudi").save } /** * Adds a method, `hoodie`, to DataFrameReader */ implicit class AvroDataFrameReader(reader: DataFrameReader) { - def avro: String => DataFrame = reader.format("com.uber.hoodie").load + def avro: String => DataFrame = reader.format("org.apache.hudi").load } } diff --git a/hoodie-spark/src/test/java/DataSourceTestUtils.java b/hudi-spark/src/test/java/DataSourceTestUtils.java similarity index 91% rename from hoodie-spark/src/test/java/DataSourceTestUtils.java rename to hudi-spark/src/test/java/DataSourceTestUtils.java index 12da7acbf..30d991856 100644 --- a/hoodie-spark/src/test/java/DataSourceTestUtils.java +++ b/hudi-spark/src/test/java/DataSourceTestUtils.java @@ -16,12 +16,12 @@ * limitations under the License. */ -import com.uber.hoodie.common.TestRawTripPayload; -import com.uber.hoodie.common.model.HoodieRecord; -import com.uber.hoodie.common.util.Option; import java.io.IOException; import java.util.List; import java.util.stream.Collectors; +import org.apache.hudi.common.TestRawTripPayload; +import org.apache.hudi.common.model.HoodieRecord; +import org.apache.hudi.common.util.Option; /** * Test utils for data source tests. diff --git a/hoodie-spark/src/test/java/HoodieJavaApp.java b/hudi-spark/src/test/java/HoodieJavaApp.java similarity index 92% rename from hoodie-spark/src/test/java/HoodieJavaApp.java rename to hudi-spark/src/test/java/HoodieJavaApp.java index eb2f68d84..ee7c6b939 100644 --- a/hoodie-spark/src/test/java/HoodieJavaApp.java +++ b/hudi-spark/src/test/java/HoodieJavaApp.java @@ -18,18 +18,18 @@ import com.beust.jcommander.JCommander; import com.beust.jcommander.Parameter; -import com.uber.hoodie.DataSourceReadOptions; -import com.uber.hoodie.DataSourceWriteOptions; -import com.uber.hoodie.HoodieDataSourceHelpers; -import com.uber.hoodie.NonpartitionedKeyGenerator; -import com.uber.hoodie.SimpleKeyGenerator; -import com.uber.hoodie.common.HoodieTestDataGenerator; -import com.uber.hoodie.common.model.HoodieTableType; -import com.uber.hoodie.config.HoodieWriteConfig; -import com.uber.hoodie.hive.MultiPartKeysValueExtractor; -import com.uber.hoodie.hive.NonPartitionedExtractor; import java.util.List; import org.apache.hadoop.fs.FileSystem; +import org.apache.hudi.DataSourceReadOptions; +import org.apache.hudi.DataSourceWriteOptions; +import org.apache.hudi.HoodieDataSourceHelpers; +import org.apache.hudi.NonpartitionedKeyGenerator; +import org.apache.hudi.SimpleKeyGenerator; +import org.apache.hudi.common.HoodieTestDataGenerator; +import org.apache.hudi.common.model.HoodieTableType; +import org.apache.hudi.config.HoodieWriteConfig; +import org.apache.hudi.hive.MultiPartKeysValueExtractor; +import org.apache.hudi.hive.NonPartitionedExtractor; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; import org.apache.spark.api.java.JavaSparkContext; @@ -121,7 +121,7 @@ public class HoodieJavaApp { Dataset inputDF1 = spark.read().json(jssc.parallelize(records1, 2)); // Save as hoodie dataset (copy on write) - DataFrameWriter writer = inputDF1.write().format("com.uber.hoodie") // specify the hoodie source + DataFrameWriter writer = inputDF1.write().format("org.apache.hudi") // specify the hoodie source .option("hoodie.insert.shuffle.parallelism", "2") // any hoodie client config can be passed like this .option("hoodie.upsert.shuffle.parallelism", @@ -154,7 +154,7 @@ public class HoodieJavaApp { List records2 = DataSourceTestUtils.convertToStringList( dataGen.generateUpdates("002"/* ignore */, 100)); Dataset inputDF2 = spark.read().json(jssc.parallelize(records2, 2)); - writer = inputDF2.write().format("com.uber.hoodie").option("hoodie.insert.shuffle.parallelism", "2") + writer = inputDF2.write().format("org.apache.hudi").option("hoodie.insert.shuffle.parallelism", "2") .option("hoodie.upsert.shuffle.parallelism", "2") .option(DataSourceWriteOptions.STORAGE_TYPE_OPT_KEY(), tableType) // Hoodie Table Type .option(DataSourceWriteOptions.RECORDKEY_FIELD_OPT_KEY(), "_row_key") @@ -173,7 +173,7 @@ public class HoodieJavaApp { /** * Read & do some queries */ - Dataset hoodieROViewDF = spark.read().format("com.uber.hoodie") + Dataset hoodieROViewDF = spark.read().format("org.apache.hudi") // pass any path glob, can include hoodie & non-hoodie // datasets .load(tablePath + (nonPartitionedTable ? "/*" : "/*/*/*/*")); @@ -187,7 +187,7 @@ public class HoodieJavaApp { /** * Consume incrementally, only changes in commit 2 above. Currently only supported for COPY_ON_WRITE TABLE */ - Dataset hoodieIncViewDF = spark.read().format("com.uber.hoodie") + Dataset hoodieIncViewDF = spark.read().format("org.apache.hudi") .option(DataSourceReadOptions.VIEW_TYPE_OPT_KEY(), DataSourceReadOptions.VIEW_TYPE_INCREMENTAL_OPT_VAL()) .option(DataSourceReadOptions.BEGIN_INSTANTTIME_OPT_KEY(), diff --git a/hoodie-spark/src/test/java/HoodieJavaStreamingApp.java b/hudi-spark/src/test/java/HoodieJavaStreamingApp.java similarity index 93% rename from hoodie-spark/src/test/java/HoodieJavaStreamingApp.java rename to hudi-spark/src/test/java/HoodieJavaStreamingApp.java index 9dd1633ee..2a0b6e228 100644 --- a/hoodie-spark/src/test/java/HoodieJavaStreamingApp.java +++ b/hudi-spark/src/test/java/HoodieJavaStreamingApp.java @@ -18,26 +18,27 @@ import com.beust.jcommander.JCommander; import com.beust.jcommander.Parameter; -import com.uber.hoodie.DataSourceReadOptions; -import com.uber.hoodie.DataSourceWriteOptions; -import com.uber.hoodie.HoodieDataSourceHelpers; -import com.uber.hoodie.common.HoodieTestDataGenerator; -import com.uber.hoodie.common.model.HoodieTableType; -import com.uber.hoodie.config.HoodieWriteConfig; -import com.uber.hoodie.hive.MultiPartKeysValueExtractor; - import java.util.List; import java.util.concurrent.Callable; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; - import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hudi.DataSourceReadOptions; +import org.apache.hudi.DataSourceWriteOptions; +import org.apache.hudi.HoodieDataSourceHelpers; +import org.apache.hudi.common.HoodieTestDataGenerator; +import org.apache.hudi.common.model.HoodieTableType; +import org.apache.hudi.config.HoodieWriteConfig; +import org.apache.hudi.hive.MultiPartKeysValueExtractor; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; import org.apache.spark.api.java.JavaSparkContext; -import org.apache.spark.sql.*; +import org.apache.spark.sql.Dataset; +import org.apache.spark.sql.Row; +import org.apache.spark.sql.SaveMode; +import org.apache.spark.sql.SparkSession; import org.apache.spark.sql.streaming.DataStreamWriter; import org.apache.spark.sql.streaming.OutputMode; import org.apache.spark.sql.streaming.ProcessingTime; @@ -198,7 +199,7 @@ public class HoodieJavaStreamingApp { /** * Read & do some queries */ - Dataset hoodieROViewDF = spark.read().format("com.uber.hoodie") + Dataset hoodieROViewDF = spark.read().format("org.apache.hudi") // pass any path glob, can include hoodie & non-hoodie // datasets .load(tablePath + "/*/*/*/*"); @@ -212,7 +213,7 @@ public class HoodieJavaStreamingApp { /** * Consume incrementally, only changes in commit 2 above. Currently only supported for COPY_ON_WRITE TABLE */ - Dataset hoodieIncViewDF = spark.read().format("com.uber.hoodie") + Dataset hoodieIncViewDF = spark.read().format("org.apache.hudi") .option(DataSourceReadOptions.VIEW_TYPE_OPT_KEY(), DataSourceReadOptions.VIEW_TYPE_INCREMENTAL_OPT_VAL()) .option(DataSourceReadOptions.BEGIN_INSTANTTIME_OPT_KEY(), @@ -234,7 +235,7 @@ public class HoodieJavaStreamingApp { DataStreamWriter writer = streamingInput .writeStream() - .format("com.uber.hoodie") + .format("org.apache.hudi") .option("hoodie.insert.shuffle.parallelism", "2") .option("hoodie.upsert.shuffle.parallelism", "2") .option(DataSourceWriteOptions.STORAGE_TYPE_OPT_KEY(), tableType) diff --git a/hoodie-spark/src/test/resources/log4j-surefire-quiet.properties b/hudi-spark/src/test/resources/log4j-surefire-quiet.properties similarity index 100% rename from hoodie-spark/src/test/resources/log4j-surefire-quiet.properties rename to hudi-spark/src/test/resources/log4j-surefire-quiet.properties diff --git a/hoodie-spark/src/test/resources/log4j-surefire.properties b/hudi-spark/src/test/resources/log4j-surefire.properties similarity index 100% rename from hoodie-spark/src/test/resources/log4j-surefire.properties rename to hudi-spark/src/test/resources/log4j-surefire.properties diff --git a/hoodie-spark/src/test/scala/DataSourceDefaultsTest.scala b/hudi-spark/src/test/scala/DataSourceDefaultsTest.scala similarity index 96% rename from hoodie-spark/src/test/scala/DataSourceDefaultsTest.scala rename to hudi-spark/src/test/scala/DataSourceDefaultsTest.scala index 8aa76b67e..f9101c5fe 100644 --- a/hoodie-spark/src/test/scala/DataSourceDefaultsTest.scala +++ b/hudi-spark/src/test/scala/DataSourceDefaultsTest.scala @@ -15,10 +15,10 @@ * limitations under the License. */ -import com.uber.hoodie.common.util.{Option, SchemaTestUtil, TypedProperties} -import com.uber.hoodie.exception.HoodieException -import com.uber.hoodie.{DataSourceWriteOptions, EmptyHoodieRecordPayload, OverwriteWithLatestAvroPayload, SimpleKeyGenerator, ComplexKeyGenerator} import org.apache.avro.generic.GenericRecord +import org.apache.hudi.common.util.{Option, SchemaTestUtil, TypedProperties} +import org.apache.hudi.exception.HoodieException +import org.apache.hudi.{ComplexKeyGenerator, DataSourceWriteOptions, EmptyHoodieRecordPayload, OverwriteWithLatestAvroPayload, SimpleKeyGenerator} import org.junit.Assert._ import org.junit.{Before, Test} import org.scalatest.junit.AssertionsForJUnit diff --git a/hoodie-spark/src/test/scala/DataSourceTest.scala b/hudi-spark/src/test/scala/DataSourceTest.scala similarity index 90% rename from hoodie-spark/src/test/scala/DataSourceTest.scala rename to hudi-spark/src/test/scala/DataSourceTest.scala index 66c0ac779..f5a00f023 100644 --- a/hoodie-spark/src/test/scala/DataSourceTest.scala +++ b/hudi-spark/src/test/scala/DataSourceTest.scala @@ -15,11 +15,11 @@ * limitations under the License. */ -import com.uber.hoodie.common.HoodieTestDataGenerator -import com.uber.hoodie.common.util.FSUtils -import com.uber.hoodie.config.HoodieWriteConfig -import com.uber.hoodie.{DataSourceReadOptions, DataSourceWriteOptions, HoodieDataSourceHelpers} import org.apache.hadoop.fs.{FileSystem, Path} +import org.apache.hudi.common.HoodieTestDataGenerator +import org.apache.hudi.common.util.FSUtils +import org.apache.hudi.config.HoodieWriteConfig +import org.apache.hudi.{DataSourceReadOptions, DataSourceWriteOptions, HoodieDataSourceHelpers} import org.apache.spark.sql._ import org.apache.spark.sql.streaming.{OutputMode, ProcessingTime} import org.junit.Assert._ @@ -67,7 +67,7 @@ class DataSourceTest extends AssertionsForJUnit { // Insert Operation val records1 = DataSourceTestUtils.convertToStringList(dataGen.generateInserts("000", 100)).toList val inputDF1: Dataset[Row] = spark.read.json(spark.sparkContext.parallelize(records1, 2)) - inputDF1.write.format("com.uber.hoodie") + inputDF1.write.format("org.apache.hudi") .options(commonOpts) .option(DataSourceWriteOptions.OPERATION_OPT_KEY, DataSourceWriteOptions.INSERT_OPERATION_OPT_VAL) .mode(SaveMode.Overwrite) @@ -77,7 +77,7 @@ class DataSourceTest extends AssertionsForJUnit { val commitInstantTime1: String = HoodieDataSourceHelpers.latestCommit(fs, basePath) // Read RO View - val hoodieROViewDF1 = spark.read.format("com.uber.hoodie") + val hoodieROViewDF1 = spark.read.format("org.apache.hudi") .load(basePath + "/*/*/*/*"); assertEquals(100, hoodieROViewDF1.count()) @@ -86,7 +86,7 @@ class DataSourceTest extends AssertionsForJUnit { val uniqueKeyCnt = inputDF2.select("_row_key").distinct().count() // Upsert Operation - inputDF2.write.format("com.uber.hoodie") + inputDF2.write.format("org.apache.hudi") .options(commonOpts) .mode(SaveMode.Append) .save(basePath) @@ -95,14 +95,14 @@ class DataSourceTest extends AssertionsForJUnit { assertEquals(2, HoodieDataSourceHelpers.listCommitsSince(fs, basePath, "000").size()) // Read RO View - val hoodieROViewDF2 = spark.read.format("com.uber.hoodie") + val hoodieROViewDF2 = spark.read.format("org.apache.hudi") .load(basePath + "/*/*/*/*"); assertEquals(100, hoodieROViewDF2.count()) // still 100, since we only updated // Read Incremental View // we have 2 commits, try pulling the first commit (which is not the latest) val firstCommit = HoodieDataSourceHelpers.listCommitsSince(fs, basePath, "000").get(0); - val hoodieIncViewDF1 = spark.read.format("com.uber.hoodie") + val hoodieIncViewDF1 = spark.read.format("org.apache.hudi") .option(DataSourceReadOptions.VIEW_TYPE_OPT_KEY, DataSourceReadOptions.VIEW_TYPE_INCREMENTAL_OPT_VAL) .option(DataSourceReadOptions.BEGIN_INSTANTTIME_OPT_KEY, "000") .option(DataSourceReadOptions.END_INSTANTTIME_OPT_KEY, firstCommit) @@ -113,7 +113,7 @@ class DataSourceTest extends AssertionsForJUnit { assertEquals(firstCommit, countsPerCommit(0).get(0)) // pull the latest commit - val hoodieIncViewDF2 = spark.read.format("com.uber.hoodie") + val hoodieIncViewDF2 = spark.read.format("org.apache.hudi") .option(DataSourceReadOptions.VIEW_TYPE_OPT_KEY, DataSourceReadOptions.VIEW_TYPE_INCREMENTAL_OPT_VAL) .option(DataSourceReadOptions.BEGIN_INSTANTTIME_OPT_KEY, commitInstantTime1) .load(basePath); @@ -128,7 +128,7 @@ class DataSourceTest extends AssertionsForJUnit { // Bulk Insert Operation val records1 = DataSourceTestUtils.convertToStringList(dataGen.generateInserts("001", 100)).toList val inputDF1: Dataset[Row] = spark.read.json(spark.sparkContext.parallelize(records1, 2)) - inputDF1.write.format("com.uber.hoodie") + inputDF1.write.format("org.apache.hudi") .options(commonOpts) .option("hoodie.compact.inline", "false") // else fails due to compaction & deltacommit instant times being same .option(DataSourceWriteOptions.OPERATION_OPT_KEY, DataSourceWriteOptions.INSERT_OPERATION_OPT_VAL) @@ -139,7 +139,7 @@ class DataSourceTest extends AssertionsForJUnit { assertTrue(HoodieDataSourceHelpers.hasNewCommits(fs, basePath, "000")) // Read RO View - val hoodieROViewDF1 = spark.read.format("com.uber.hoodie").load(basePath + "/*/*/*/*") + val hoodieROViewDF1 = spark.read.format("org.apache.hudi").load(basePath + "/*/*/*/*") assertEquals(100, hoodieROViewDF1.count()) // still 100, since we only updated } @@ -156,12 +156,12 @@ class DataSourceTest extends AssertionsForJUnit { val records1 = DataSourceTestUtils.convertToStringList(inserts1).toList val inputDF1: Dataset[Row] = spark.read.json(spark.sparkContext.parallelize(records1, 2)) - inputDF1.write.format("com.uber.hoodie") + inputDF1.write.format("org.apache.hudi") .options(commonOpts) .option(DataSourceWriteOptions.OPERATION_OPT_KEY, DataSourceWriteOptions.INSERT_OPERATION_OPT_VAL) .mode(SaveMode.Overwrite) .save(basePath) - val hoodieROViewDF1 = spark.read.format("com.uber.hoodie") + val hoodieROViewDF1 = spark.read.format("org.apache.hudi") .load(basePath + "/*/*/*/*") assertEquals(insert1Cnt, hoodieROViewDF1.count()) @@ -170,16 +170,16 @@ class DataSourceTest extends AssertionsForJUnit { .convertToStringList(inserts2Dup ++ inserts2New) .toList val inputDF2: Dataset[Row] = spark.read.json(spark.sparkContext.parallelize(records2, 2)) - inputDF2.write.format("com.uber.hoodie") + inputDF2.write.format("org.apache.hudi") .options(commonOpts) .option(DataSourceWriteOptions.INSERT_DROP_DUPS_OPT_KEY, "true") .mode(SaveMode.Append) .save(basePath) - val hoodieROViewDF2 = spark.read.format("com.uber.hoodie") + val hoodieROViewDF2 = spark.read.format("org.apache.hudi") .load(basePath + "/*/*/*/*") assertEquals(hoodieROViewDF2.count(), totalUniqueKeyToGenerate) - val hoodieIncViewDF2 = spark.read.format("com.uber.hoodie") + val hoodieIncViewDF2 = spark.read.format("org.apache.hudi") .option(DataSourceReadOptions.VIEW_TYPE_OPT_KEY, DataSourceReadOptions.VIEW_TYPE_INCREMENTAL_OPT_VAL) .option(DataSourceReadOptions.BEGIN_INSTANTTIME_OPT_KEY, commitInstantTime1) .load(basePath) @@ -213,7 +213,7 @@ class DataSourceTest extends AssertionsForJUnit { //'writeStream' can be called only on streaming Dataset/DataFrame streamingInput .writeStream - .format("com.uber.hoodie") + .format("org.apache.hudi") .options(commonOpts) .trigger(new ProcessingTime(100)) .option("checkpointLocation", basePath + "/checkpoint") @@ -230,7 +230,7 @@ class DataSourceTest extends AssertionsForJUnit { assertTrue(HoodieDataSourceHelpers.hasNewCommits(fs, destPath, "000")) val commitInstantTime1: String = HoodieDataSourceHelpers.latestCommit(fs, destPath) // Read RO View - val hoodieROViewDF1 = spark.read.format("com.uber.hoodie") + val hoodieROViewDF1 = spark.read.format("org.apache.hudi") .load(destPath + "/*/*/*/*") assert(hoodieROViewDF1.count() == 100) @@ -241,7 +241,7 @@ class DataSourceTest extends AssertionsForJUnit { assertEquals(2, HoodieDataSourceHelpers.listCommitsSince(fs, destPath, "000").size()) // Read RO View - val hoodieROViewDF2 = spark.read.format("com.uber.hoodie") + val hoodieROViewDF2 = spark.read.format("org.apache.hudi") .load(destPath + "/*/*/*/*") assertEquals(100, hoodieROViewDF2.count()) // still 100, since we only updated @@ -249,7 +249,7 @@ class DataSourceTest extends AssertionsForJUnit { // Read Incremental View // we have 2 commits, try pulling the first commit (which is not the latest) val firstCommit = HoodieDataSourceHelpers.listCommitsSince(fs, destPath, "000").get(0) - val hoodieIncViewDF1 = spark.read.format("com.uber.hoodie") + val hoodieIncViewDF1 = spark.read.format("org.apache.hudi") .option(DataSourceReadOptions.VIEW_TYPE_OPT_KEY, DataSourceReadOptions.VIEW_TYPE_INCREMENTAL_OPT_VAL) .option(DataSourceReadOptions.BEGIN_INSTANTTIME_OPT_KEY, "000") .option(DataSourceReadOptions.END_INSTANTTIME_OPT_KEY, firstCommit) @@ -261,7 +261,7 @@ class DataSourceTest extends AssertionsForJUnit { assertEquals(firstCommit, countsPerCommit(0).get(0)) // pull the latest commit - val hoodieIncViewDF2 = spark.read.format("com.uber.hoodie") + val hoodieIncViewDF2 = spark.read.format("org.apache.hudi") .option(DataSourceReadOptions.VIEW_TYPE_OPT_KEY, DataSourceReadOptions.VIEW_TYPE_INCREMENTAL_OPT_VAL) .option(DataSourceReadOptions.BEGIN_INSTANTTIME_OPT_KEY, commitInstantTime1) .load(destPath) diff --git a/hoodie-spark/src/test/scala/com/uber/hoodie/HoodieSparkSqlWriterSuite.scala b/hudi-spark/src/test/scala/org/apache/hudi/HoodieSparkSqlWriterSuite.scala similarity index 93% rename from hoodie-spark/src/test/scala/com/uber/hoodie/HoodieSparkSqlWriterSuite.scala rename to hudi-spark/src/test/scala/org/apache/hudi/HoodieSparkSqlWriterSuite.scala index 28448d12b..be369712c 100644 --- a/hoodie-spark/src/test/scala/com/uber/hoodie/HoodieSparkSqlWriterSuite.scala +++ b/hudi-spark/src/test/scala/org/apache/hudi/HoodieSparkSqlWriterSuite.scala @@ -15,13 +15,13 @@ * limitations under the License. */ -package com.uber.hoodie +package org.apache.hudi -import org.scalatest.{FunSuite, Matchers} -import DataSourceWriteOptions._ -import com.uber.hoodie.config.HoodieWriteConfig -import com.uber.hoodie.exception.HoodieException +import org.apache.hudi.DataSourceWriteOptions._ +import org.apache.hudi.config.HoodieWriteConfig +import org.apache.hudi.exception.HoodieException import org.apache.spark.sql.{SaveMode, SparkSession} +import org.scalatest.{FunSuite, Matchers} class HoodieSparkSqlWriterSuite extends FunSuite with Matchers { diff --git a/hoodie-timeline-service/pom.xml b/hudi-timeline-service/pom.xml similarity index 94% rename from hoodie-timeline-service/pom.xml rename to hudi-timeline-service/pom.xml index c5bd657da..f15833880 100644 --- a/hoodie-timeline-service/pom.xml +++ b/hudi-timeline-service/pom.xml @@ -17,13 +17,13 @@ --> - hoodie - com.uber.hoodie - 0.4.8-SNAPSHOT + hudi + org.apache.hudi + 0.5.0-SNAPSHOT 4.0.0 - hoodie-timeline-service + hudi-timeline-service jar @@ -101,7 +101,7 @@ - com.uber.hoodie.timeline.service.TimelineService + org.apache.hudi.timeline.service.TimelineService @@ -132,8 +132,8 @@ 4.3.2 - com.uber.hoodie - hoodie-common + org.apache.hudi + hudi-common ${project.version} @@ -195,8 +195,8 @@ - com.uber.hoodie - hoodie-common + org.apache.hudi + hudi-common ${project.version} tests test-jar diff --git a/hoodie-timeline-service/run_server.sh b/hudi-timeline-service/run_server.sh similarity index 100% rename from hoodie-timeline-service/run_server.sh rename to hudi-timeline-service/run_server.sh diff --git a/hoodie-timeline-service/src/main/java/com/uber/hoodie/timeline/service/FileSystemViewHandler.java b/hudi-timeline-service/src/main/java/org/apache/hudi/timeline/service/FileSystemViewHandler.java similarity index 53% rename from hoodie-timeline-service/src/main/java/com/uber/hoodie/timeline/service/FileSystemViewHandler.java rename to hudi-timeline-service/src/main/java/org/apache/hudi/timeline/service/FileSystemViewHandler.java index 483bc6f52..a19a2749b 100644 --- a/hoodie-timeline-service/src/main/java/com/uber/hoodie/timeline/service/FileSystemViewHandler.java +++ b/hudi-timeline-service/src/main/java/org/apache/hudi/timeline/service/FileSystemViewHandler.java @@ -16,25 +16,11 @@ * limitations under the License. */ -package com.uber.hoodie.timeline.service; - -import static com.uber.hoodie.common.table.view.RemoteHoodieTableFileSystemView.*; +package org.apache.hudi.timeline.service; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Preconditions; -import com.uber.hoodie.common.table.HoodieTimeline; -import com.uber.hoodie.common.table.SyncableFileSystemView; -import com.uber.hoodie.common.table.timeline.dto.CompactionOpDTO; -import com.uber.hoodie.common.table.timeline.dto.DataFileDTO; -import com.uber.hoodie.common.table.timeline.dto.FileGroupDTO; -import com.uber.hoodie.common.table.timeline.dto.FileSliceDTO; -import com.uber.hoodie.common.table.timeline.dto.InstantDTO; -import com.uber.hoodie.common.table.timeline.dto.TimelineDTO; -import com.uber.hoodie.common.table.view.FileSystemViewManager; -import com.uber.hoodie.timeline.service.handlers.DataFileHandler; -import com.uber.hoodie.timeline.service.handlers.FileSliceHandler; -import com.uber.hoodie.timeline.service.handlers.TimelineHandler; import io.javalin.Context; import io.javalin.Handler; import io.javalin.Javalin; @@ -43,6 +29,19 @@ import java.util.Arrays; import java.util.List; import java.util.stream.Collectors; import org.apache.hadoop.conf.Configuration; +import org.apache.hudi.common.table.HoodieTimeline; +import org.apache.hudi.common.table.SyncableFileSystemView; +import org.apache.hudi.common.table.timeline.dto.CompactionOpDTO; +import org.apache.hudi.common.table.timeline.dto.DataFileDTO; +import org.apache.hudi.common.table.timeline.dto.FileGroupDTO; +import org.apache.hudi.common.table.timeline.dto.FileSliceDTO; +import org.apache.hudi.common.table.timeline.dto.InstantDTO; +import org.apache.hudi.common.table.timeline.dto.TimelineDTO; +import org.apache.hudi.common.table.view.FileSystemViewManager; +import org.apache.hudi.common.table.view.RemoteHoodieTableFileSystemView; +import org.apache.hudi.timeline.service.handlers.DataFileHandler; +import org.apache.hudi.timeline.service.handlers.FileSliceHandler; +import org.apache.hudi.timeline.service.handlers.TimelineHandler; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; import org.jetbrains.annotations.NotNull; @@ -79,13 +78,12 @@ public class FileSystemViewHandler { /** * Determines if local view of dataset's timeline is behind that of client's view - * @param ctx - * @return */ private boolean isLocalViewBehind(Context ctx) { - String basePath = ctx.queryParam(BASEPATH_PARAM); - String lastKnownInstantFromClient = ctx.queryParam(LAST_INSTANT_TS, HoodieTimeline.INVALID_INSTANT_TS); - String timelineHashFromClient = ctx.queryParam(TIMELINE_HASH, ""); + String basePath = ctx.queryParam(RemoteHoodieTableFileSystemView.BASEPATH_PARAM); + String lastKnownInstantFromClient = ctx + .queryParam(RemoteHoodieTableFileSystemView.LAST_INSTANT_TS, HoodieTimeline.INVALID_INSTANT_TS); + String timelineHashFromClient = ctx.queryParam(RemoteHoodieTableFileSystemView.TIMELINE_HASH, ""); HoodieTimeline localTimeline = viewManager.getFileSystemView(basePath).getTimeline() .filterCompletedAndCompactionInstants(); if (logger.isDebugEnabled()) { @@ -110,13 +108,12 @@ public class FileSystemViewHandler { /** * Syncs data-set view if local view is behind - * @param ctx */ private boolean syncIfLocalViewBehind(Context ctx) { if (isLocalViewBehind(ctx)) { - String basePath = ctx.queryParam(BASEPATH_PARAM); + String basePath = ctx.queryParam(RemoteHoodieTableFileSystemView.BASEPATH_PARAM); String lastKnownInstantFromClient = - ctx.queryParam(LAST_INSTANT_TS, HoodieTimeline.INVALID_INSTANT_TS); + ctx.queryParam(RemoteHoodieTableFileSystemView.LAST_INSTANT_TS, HoodieTimeline.INVALID_INSTANT_TS); SyncableFileSystemView view = viewManager.getFileSystemView(basePath); synchronized (view) { if (isLocalViewBehind(ctx)) { @@ -146,13 +143,15 @@ public class FileSystemViewHandler { * Register Timeline API calls */ private void registerTimelineAPI() { - app.get(LAST_INSTANT, new ViewHandler(ctx -> { - List dtos = instantHandler.getLastInstant(ctx.validatedQueryParam(BASEPATH_PARAM).getValue()); + app.get(RemoteHoodieTableFileSystemView.LAST_INSTANT, new ViewHandler(ctx -> { + List dtos = instantHandler.getLastInstant(ctx.validatedQueryParam( + RemoteHoodieTableFileSystemView.BASEPATH_PARAM).getValue()); writeValueAsString(ctx, dtos); }, false)); - app.get(TIMELINE, new ViewHandler(ctx -> { - TimelineDTO dto = instantHandler.getTimeline(ctx.validatedQueryParam(BASEPATH_PARAM).getValue()); + app.get(RemoteHoodieTableFileSystemView.TIMELINE, new ViewHandler(ctx -> { + TimelineDTO dto = instantHandler.getTimeline(ctx.validatedQueryParam( + RemoteHoodieTableFileSystemView.BASEPATH_PARAM).getValue()); writeValueAsString(ctx, dto); }, false)); } @@ -161,54 +160,63 @@ public class FileSystemViewHandler { * Register Data-Files API calls */ private void registerDataFilesAPI() { - app.get(LATEST_PARTITION_DATA_FILES_URL, new ViewHandler(ctx -> { + app.get(RemoteHoodieTableFileSystemView.LATEST_PARTITION_DATA_FILES_URL, new ViewHandler(ctx -> { List dtos = - dataFileHandler.getLatestDataFiles(ctx.validatedQueryParam(BASEPATH_PARAM).getOrThrow(), - ctx.validatedQueryParam(PARTITION_PARAM).getOrThrow()); + dataFileHandler + .getLatestDataFiles(ctx.validatedQueryParam(RemoteHoodieTableFileSystemView.BASEPATH_PARAM).getOrThrow(), + ctx.validatedQueryParam(RemoteHoodieTableFileSystemView.PARTITION_PARAM).getOrThrow()); writeValueAsString(ctx, dtos); }, true)); - app.get(LATEST_PARTITION_DATA_FILE_URL, new ViewHandler(ctx -> { + app.get(RemoteHoodieTableFileSystemView.LATEST_PARTITION_DATA_FILE_URL, new ViewHandler(ctx -> { List dtos = - dataFileHandler.getLatestDataFile(ctx.validatedQueryParam(BASEPATH_PARAM).getOrThrow(), - ctx.validatedQueryParam(PARTITION_PARAM).getOrThrow(), - ctx.validatedQueryParam(FILEID_PARAM).getOrThrow()); + dataFileHandler + .getLatestDataFile(ctx.validatedQueryParam(RemoteHoodieTableFileSystemView.BASEPATH_PARAM).getOrThrow(), + ctx.validatedQueryParam(RemoteHoodieTableFileSystemView.PARTITION_PARAM).getOrThrow(), + ctx.validatedQueryParam(RemoteHoodieTableFileSystemView.FILEID_PARAM).getOrThrow()); writeValueAsString(ctx, dtos); }, true)); - app.get(LATEST_ALL_DATA_FILES, new ViewHandler(ctx -> { + app.get(RemoteHoodieTableFileSystemView.LATEST_ALL_DATA_FILES, new ViewHandler(ctx -> { List dtos = - dataFileHandler.getLatestDataFiles(ctx.validatedQueryParam(BASEPATH_PARAM).getOrThrow()); + dataFileHandler + .getLatestDataFiles(ctx.validatedQueryParam(RemoteHoodieTableFileSystemView.BASEPATH_PARAM).getOrThrow()); writeValueAsString(ctx, dtos); }, true)); - app.get(LATEST_DATA_FILES_BEFORE_ON_INSTANT_URL, new ViewHandler(ctx -> { + app.get(RemoteHoodieTableFileSystemView.LATEST_DATA_FILES_BEFORE_ON_INSTANT_URL, new ViewHandler(ctx -> { List dtos = - dataFileHandler.getLatestDataFilesBeforeOrOn(ctx.validatedQueryParam(BASEPATH_PARAM).getOrThrow(), - ctx.validatedQueryParam(PARTITION_PARAM).getOrThrow(), - ctx.validatedQueryParam(MAX_INSTANT_PARAM).getOrThrow()); + dataFileHandler.getLatestDataFilesBeforeOrOn(ctx.validatedQueryParam( + RemoteHoodieTableFileSystemView.BASEPATH_PARAM).getOrThrow(), + ctx.validatedQueryParam(RemoteHoodieTableFileSystemView.PARTITION_PARAM).getOrThrow(), + ctx.validatedQueryParam(RemoteHoodieTableFileSystemView.MAX_INSTANT_PARAM).getOrThrow()); writeValueAsString(ctx, dtos); }, true)); - app.get(LATEST_DATA_FILE_ON_INSTANT_URL, new ViewHandler(ctx -> { + app.get(RemoteHoodieTableFileSystemView.LATEST_DATA_FILE_ON_INSTANT_URL, new ViewHandler(ctx -> { List dtos = - dataFileHandler.getLatestDataFileOn(ctx.validatedQueryParam(BASEPATH_PARAM).getOrThrow(), - ctx.validatedQueryParam(PARTITION_PARAM).getOrThrow(), - ctx.queryParam(INSTANT_PARAM), ctx.validatedQueryParam(FILEID_PARAM).getOrThrow()); + dataFileHandler + .getLatestDataFileOn(ctx.validatedQueryParam(RemoteHoodieTableFileSystemView.BASEPATH_PARAM).getOrThrow(), + ctx.validatedQueryParam(RemoteHoodieTableFileSystemView.PARTITION_PARAM).getOrThrow(), + ctx.queryParam(RemoteHoodieTableFileSystemView.INSTANT_PARAM), ctx.validatedQueryParam( + RemoteHoodieTableFileSystemView.FILEID_PARAM).getOrThrow()); writeValueAsString(ctx, dtos); }, true)); - app.get(ALL_DATA_FILES, new ViewHandler(ctx -> { + app.get(RemoteHoodieTableFileSystemView.ALL_DATA_FILES, new ViewHandler(ctx -> { List dtos = - dataFileHandler.getAllDataFiles(ctx.validatedQueryParam(BASEPATH_PARAM).getOrThrow(), - ctx.validatedQueryParam(PARTITION_PARAM).getOrThrow()); + dataFileHandler + .getAllDataFiles(ctx.validatedQueryParam(RemoteHoodieTableFileSystemView.BASEPATH_PARAM).getOrThrow(), + ctx.validatedQueryParam(RemoteHoodieTableFileSystemView.PARTITION_PARAM).getOrThrow()); writeValueAsString(ctx, dtos); }, true)); - app.get(LATEST_DATA_FILES_RANGE_INSTANT_URL, new ViewHandler(ctx -> { + app.get(RemoteHoodieTableFileSystemView.LATEST_DATA_FILES_RANGE_INSTANT_URL, new ViewHandler(ctx -> { List dtos = - dataFileHandler.getLatestDataFilesInRange(ctx.validatedQueryParam(BASEPATH_PARAM).getOrThrow(), - Arrays.asList(ctx.validatedQueryParam(INSTANTS_PARAM).getOrThrow().split(","))); + dataFileHandler.getLatestDataFilesInRange(ctx.validatedQueryParam( + RemoteHoodieTableFileSystemView.BASEPATH_PARAM).getOrThrow(), + Arrays.asList( + ctx.validatedQueryParam(RemoteHoodieTableFileSystemView.INSTANTS_PARAM).getOrThrow().split(","))); writeValueAsString(ctx, dtos); }, true)); } @@ -217,79 +225,90 @@ public class FileSystemViewHandler { * Register File Slices API calls */ private void registerFileSlicesAPI() { - app.get(LATEST_PARTITION_SLICES_URL, new ViewHandler(ctx -> { + app.get(RemoteHoodieTableFileSystemView.LATEST_PARTITION_SLICES_URL, new ViewHandler(ctx -> { List dtos = - sliceHandler.getLatestFileSlices(ctx.validatedQueryParam(BASEPATH_PARAM).getOrThrow(), - ctx.validatedQueryParam(PARTITION_PARAM).getOrThrow()); + sliceHandler + .getLatestFileSlices(ctx.validatedQueryParam(RemoteHoodieTableFileSystemView.BASEPATH_PARAM).getOrThrow(), + ctx.validatedQueryParam(RemoteHoodieTableFileSystemView.PARTITION_PARAM).getOrThrow()); writeValueAsString(ctx, dtos); }, true)); - app.get(LATEST_PARTITION_SLICE_URL, new ViewHandler(ctx -> { + app.get(RemoteHoodieTableFileSystemView.LATEST_PARTITION_SLICE_URL, new ViewHandler(ctx -> { List dtos = - sliceHandler.getLatestFileSlice(ctx.validatedQueryParam(BASEPATH_PARAM).getOrThrow(), - ctx.validatedQueryParam(PARTITION_PARAM).getOrThrow(), - ctx.validatedQueryParam(FILEID_PARAM).getOrThrow()); + sliceHandler + .getLatestFileSlice(ctx.validatedQueryParam(RemoteHoodieTableFileSystemView.BASEPATH_PARAM).getOrThrow(), + ctx.validatedQueryParam(RemoteHoodieTableFileSystemView.PARTITION_PARAM).getOrThrow(), + ctx.validatedQueryParam(RemoteHoodieTableFileSystemView.FILEID_PARAM).getOrThrow()); writeValueAsString(ctx, dtos); }, true)); - app.get(LATEST_PARTITION_UNCOMPACTED_SLICES_URL, new ViewHandler(ctx -> { + app.get(RemoteHoodieTableFileSystemView.LATEST_PARTITION_UNCOMPACTED_SLICES_URL, new ViewHandler(ctx -> { List dtos = - sliceHandler.getLatestUnCompactedFileSlices(ctx.validatedQueryParam(BASEPATH_PARAM).getOrThrow(), - ctx.validatedQueryParam(PARTITION_PARAM).getOrThrow()); + sliceHandler.getLatestUnCompactedFileSlices(ctx.validatedQueryParam( + RemoteHoodieTableFileSystemView.BASEPATH_PARAM).getOrThrow(), + ctx.validatedQueryParam(RemoteHoodieTableFileSystemView.PARTITION_PARAM).getOrThrow()); writeValueAsString(ctx, dtos); }, true)); - app.get(ALL_SLICES_URL, new ViewHandler(ctx -> { + app.get(RemoteHoodieTableFileSystemView.ALL_SLICES_URL, new ViewHandler(ctx -> { List dtos = - sliceHandler.getAllFileSlices(ctx.validatedQueryParam(BASEPATH_PARAM).getOrThrow(), - ctx.validatedQueryParam(PARTITION_PARAM).getOrThrow()); + sliceHandler + .getAllFileSlices(ctx.validatedQueryParam(RemoteHoodieTableFileSystemView.BASEPATH_PARAM).getOrThrow(), + ctx.validatedQueryParam(RemoteHoodieTableFileSystemView.PARTITION_PARAM).getOrThrow()); writeValueAsString(ctx, dtos); }, true)); - app.get(LATEST_SLICES_RANGE_INSTANT_URL, new ViewHandler(ctx -> { + app.get(RemoteHoodieTableFileSystemView.LATEST_SLICES_RANGE_INSTANT_URL, new ViewHandler(ctx -> { List dtos = - sliceHandler.getLatestFileSliceInRange(ctx.validatedQueryParam(BASEPATH_PARAM).getOrThrow(), - Arrays.asList(ctx.validatedQueryParam(INSTANTS_PARAM).getOrThrow().split(","))); + sliceHandler.getLatestFileSliceInRange( + ctx.validatedQueryParam(RemoteHoodieTableFileSystemView.BASEPATH_PARAM).getOrThrow(), + Arrays.asList( + ctx.validatedQueryParam(RemoteHoodieTableFileSystemView.INSTANTS_PARAM).getOrThrow().split(","))); writeValueAsString(ctx, dtos); }, true)); - app.get(LATEST_SLICES_MERGED_BEFORE_ON_INSTANT_URL, new ViewHandler(ctx -> { + app.get(RemoteHoodieTableFileSystemView.LATEST_SLICES_MERGED_BEFORE_ON_INSTANT_URL, new ViewHandler(ctx -> { List dtos = - sliceHandler.getLatestMergedFileSlicesBeforeOrOn(ctx.validatedQueryParam(BASEPATH_PARAM).getOrThrow(), - ctx.validatedQueryParam(PARTITION_PARAM).getOrThrow(), - ctx.validatedQueryParam(MAX_INSTANT_PARAM).getOrThrow()); + sliceHandler.getLatestMergedFileSlicesBeforeOrOn(ctx.validatedQueryParam( + RemoteHoodieTableFileSystemView.BASEPATH_PARAM).getOrThrow(), + ctx.validatedQueryParam(RemoteHoodieTableFileSystemView.PARTITION_PARAM).getOrThrow(), + ctx.validatedQueryParam(RemoteHoodieTableFileSystemView.MAX_INSTANT_PARAM).getOrThrow()); writeValueAsString(ctx, dtos); }, true)); - app.get(LATEST_SLICES_BEFORE_ON_INSTANT_URL, new ViewHandler(ctx -> { + app.get(RemoteHoodieTableFileSystemView.LATEST_SLICES_BEFORE_ON_INSTANT_URL, new ViewHandler(ctx -> { List dtos = - sliceHandler.getLatestFileSlicesBeforeOrOn(ctx.validatedQueryParam(BASEPATH_PARAM).getOrThrow(), - ctx.validatedQueryParam(PARTITION_PARAM).getOrThrow(), - ctx.validatedQueryParam(MAX_INSTANT_PARAM).getOrThrow(), - Boolean.valueOf(ctx.validatedQueryParam(INCLUDE_FILES_IN_PENDING_COMPACTION_PARAM).getOrThrow())); + sliceHandler.getLatestFileSlicesBeforeOrOn(ctx.validatedQueryParam( + RemoteHoodieTableFileSystemView.BASEPATH_PARAM).getOrThrow(), + ctx.validatedQueryParam(RemoteHoodieTableFileSystemView.PARTITION_PARAM).getOrThrow(), + ctx.validatedQueryParam(RemoteHoodieTableFileSystemView.MAX_INSTANT_PARAM).getOrThrow(), + Boolean.valueOf(ctx.validatedQueryParam( + RemoteHoodieTableFileSystemView.INCLUDE_FILES_IN_PENDING_COMPACTION_PARAM).getOrThrow())); writeValueAsString(ctx, dtos); }, true)); - app.get(PENDING_COMPACTION_OPS, new ViewHandler(ctx -> { + app.get(RemoteHoodieTableFileSystemView.PENDING_COMPACTION_OPS, new ViewHandler(ctx -> { List dtos = sliceHandler.getPendingCompactionOperations( - ctx.validatedQueryParam(BASEPATH_PARAM).getOrThrow()); + ctx.validatedQueryParam(RemoteHoodieTableFileSystemView.BASEPATH_PARAM).getOrThrow()); writeValueAsString(ctx, dtos); }, true)); - app.get(ALL_FILEGROUPS_FOR_PARTITION_URL, new ViewHandler(ctx -> { - List dtos = sliceHandler.getAllFileGroups(ctx.validatedQueryParam(BASEPATH_PARAM).getOrThrow(), - ctx.validatedQueryParam(PARTITION_PARAM).getOrThrow()); + app.get(RemoteHoodieTableFileSystemView.ALL_FILEGROUPS_FOR_PARTITION_URL, new ViewHandler(ctx -> { + List dtos = sliceHandler.getAllFileGroups(ctx.validatedQueryParam( + RemoteHoodieTableFileSystemView.BASEPATH_PARAM).getOrThrow(), + ctx.validatedQueryParam(RemoteHoodieTableFileSystemView.PARTITION_PARAM).getOrThrow()); writeValueAsString(ctx, dtos); }, true)); - app.post(REFRESH_DATASET, new ViewHandler(ctx -> { - boolean success = sliceHandler.refreshDataset(ctx.validatedQueryParam(BASEPATH_PARAM).getOrThrow()); + app.post(RemoteHoodieTableFileSystemView.REFRESH_DATASET, new ViewHandler(ctx -> { + boolean success = sliceHandler.refreshDataset(ctx.validatedQueryParam( + RemoteHoodieTableFileSystemView.BASEPATH_PARAM).getOrThrow()); writeValueAsString(ctx, success); }, false)); } private static boolean isRefreshCheckDisabledInQuery(Context ctxt) { - return Boolean.valueOf(ctxt.queryParam(REFRESH_OFF)); + return Boolean.valueOf(ctxt.queryParam(RemoteHoodieTableFileSystemView.REFRESH_OFF)); } /** @@ -330,9 +349,9 @@ public class FileSystemViewHandler { if (refreshCheck) { long beginFinalCheck = System.currentTimeMillis(); String errMsg = "Last known instant from client was " - + context.queryParam(LAST_INSTANT_TS, HoodieTimeline.INVALID_INSTANT_TS) + + context.queryParam(RemoteHoodieTableFileSystemView.LAST_INSTANT_TS, HoodieTimeline.INVALID_INSTANT_TS) + " but server has the following timeline " - + viewManager.getFileSystemView(context.queryParam(BASEPATH_PARAM)) + + viewManager.getFileSystemView(context.queryParam(RemoteHoodieTableFileSystemView.BASEPATH_PARAM)) .getTimeline().getInstants().collect(Collectors.toList()); Preconditions.checkArgument(!isLocalViewBehind(context), errMsg); long endFinalCheck = System.currentTimeMillis(); diff --git a/hoodie-timeline-service/src/main/java/com/uber/hoodie/timeline/service/TimelineService.java b/hudi-timeline-service/src/main/java/org/apache/hudi/timeline/service/TimelineService.java similarity index 95% rename from hoodie-timeline-service/src/main/java/com/uber/hoodie/timeline/service/TimelineService.java rename to hudi-timeline-service/src/main/java/org/apache/hudi/timeline/service/TimelineService.java index 0707badb5..392c0331e 100644 --- a/hoodie-timeline-service/src/main/java/com/uber/hoodie/timeline/service/TimelineService.java +++ b/hudi-timeline-service/src/main/java/org/apache/hudi/timeline/service/TimelineService.java @@ -16,20 +16,20 @@ * limitations under the License. */ -package com.uber.hoodie.timeline.service; +package org.apache.hudi.timeline.service; import com.beust.jcommander.JCommander; import com.beust.jcommander.Parameter; -import com.uber.hoodie.common.SerializableConfiguration; -import com.uber.hoodie.common.table.view.FileSystemViewManager; -import com.uber.hoodie.common.table.view.FileSystemViewStorageConfig; -import com.uber.hoodie.common.table.view.FileSystemViewStorageType; -import com.uber.hoodie.common.util.FSUtils; import io.javalin.Javalin; import java.io.IOException; import java.io.Serializable; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; +import org.apache.hudi.common.SerializableConfiguration; +import org.apache.hudi.common.table.view.FileSystemViewManager; +import org.apache.hudi.common.table.view.FileSystemViewStorageConfig; +import org.apache.hudi.common.table.view.FileSystemViewStorageType; +import org.apache.hudi.common.util.FSUtils; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; diff --git a/hoodie-timeline-service/src/main/java/com/uber/hoodie/timeline/service/handlers/DataFileHandler.java b/hudi-timeline-service/src/main/java/org/apache/hudi/timeline/service/handlers/DataFileHandler.java similarity index 94% rename from hoodie-timeline-service/src/main/java/com/uber/hoodie/timeline/service/handlers/DataFileHandler.java rename to hudi-timeline-service/src/main/java/org/apache/hudi/timeline/service/handlers/DataFileHandler.java index 05bb06950..a1cb8ce8c 100644 --- a/hoodie-timeline-service/src/main/java/com/uber/hoodie/timeline/service/handlers/DataFileHandler.java +++ b/hudi-timeline-service/src/main/java/org/apache/hudi/timeline/service/handlers/DataFileHandler.java @@ -16,16 +16,16 @@ * limitations under the License. */ -package com.uber.hoodie.timeline.service.handlers; +package org.apache.hudi.timeline.service.handlers; -import com.uber.hoodie.common.table.timeline.dto.DataFileDTO; -import com.uber.hoodie.common.table.view.FileSystemViewManager; import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; import java.util.List; import java.util.stream.Collectors; import org.apache.hadoop.conf.Configuration; +import org.apache.hudi.common.table.timeline.dto.DataFileDTO; +import org.apache.hudi.common.table.view.FileSystemViewManager; /** * REST Handler servicing data-file requests diff --git a/hoodie-timeline-service/src/main/java/com/uber/hoodie/timeline/service/handlers/FileSliceHandler.java b/hudi-timeline-service/src/main/java/org/apache/hudi/timeline/service/handlers/FileSliceHandler.java similarity index 92% rename from hoodie-timeline-service/src/main/java/com/uber/hoodie/timeline/service/handlers/FileSliceHandler.java rename to hudi-timeline-service/src/main/java/org/apache/hudi/timeline/service/handlers/FileSliceHandler.java index 01aceae2b..d63d31739 100644 --- a/hoodie-timeline-service/src/main/java/com/uber/hoodie/timeline/service/handlers/FileSliceHandler.java +++ b/hudi-timeline-service/src/main/java/org/apache/hudi/timeline/service/handlers/FileSliceHandler.java @@ -16,18 +16,18 @@ * limitations under the License. */ -package com.uber.hoodie.timeline.service.handlers; +package org.apache.hudi.timeline.service.handlers; -import com.uber.hoodie.common.table.timeline.dto.CompactionOpDTO; -import com.uber.hoodie.common.table.timeline.dto.FileGroupDTO; -import com.uber.hoodie.common.table.timeline.dto.FileSliceDTO; -import com.uber.hoodie.common.table.view.FileSystemViewManager; import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; import java.util.List; import java.util.stream.Collectors; import org.apache.hadoop.conf.Configuration; +import org.apache.hudi.common.table.timeline.dto.CompactionOpDTO; +import org.apache.hudi.common.table.timeline.dto.FileGroupDTO; +import org.apache.hudi.common.table.timeline.dto.FileSliceDTO; +import org.apache.hudi.common.table.view.FileSystemViewManager; /** * REST Handler servicing file-slice requests diff --git a/hoodie-timeline-service/src/main/java/com/uber/hoodie/timeline/service/handlers/Handler.java b/hudi-timeline-service/src/main/java/org/apache/hudi/timeline/service/handlers/Handler.java similarity index 91% rename from hoodie-timeline-service/src/main/java/com/uber/hoodie/timeline/service/handlers/Handler.java rename to hudi-timeline-service/src/main/java/org/apache/hudi/timeline/service/handlers/Handler.java index aef9f7350..03be706fe 100644 --- a/hoodie-timeline-service/src/main/java/com/uber/hoodie/timeline/service/handlers/Handler.java +++ b/hudi-timeline-service/src/main/java/org/apache/hudi/timeline/service/handlers/Handler.java @@ -16,12 +16,12 @@ * limitations under the License. */ -package com.uber.hoodie.timeline.service.handlers; +package org.apache.hudi.timeline.service.handlers; -import com.uber.hoodie.common.table.view.FileSystemViewManager; import java.io.IOException; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; +import org.apache.hudi.common.table.view.FileSystemViewManager; public abstract class Handler { diff --git a/hoodie-timeline-service/src/main/java/com/uber/hoodie/timeline/service/handlers/TimelineHandler.java b/hudi-timeline-service/src/main/java/org/apache/hudi/timeline/service/handlers/TimelineHandler.java similarity index 86% rename from hoodie-timeline-service/src/main/java/com/uber/hoodie/timeline/service/handlers/TimelineHandler.java rename to hudi-timeline-service/src/main/java/org/apache/hudi/timeline/service/handlers/TimelineHandler.java index e2cb7c2e1..1dfb5d6fe 100644 --- a/hoodie-timeline-service/src/main/java/com/uber/hoodie/timeline/service/handlers/TimelineHandler.java +++ b/hudi-timeline-service/src/main/java/org/apache/hudi/timeline/service/handlers/TimelineHandler.java @@ -16,16 +16,16 @@ * limitations under the License. */ -package com.uber.hoodie.timeline.service.handlers; +package org.apache.hudi.timeline.service.handlers; -import com.uber.hoodie.common.table.timeline.dto.InstantDTO; -import com.uber.hoodie.common.table.timeline.dto.TimelineDTO; -import com.uber.hoodie.common.table.view.FileSystemViewManager; import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; import java.util.List; import org.apache.hadoop.conf.Configuration; +import org.apache.hudi.common.table.timeline.dto.InstantDTO; +import org.apache.hudi.common.table.timeline.dto.TimelineDTO; +import org.apache.hudi.common.table.view.FileSystemViewManager; /** * REST Handler servicing timeline requests diff --git a/hoodie-timeline-service/src/main/resources/META-INF/LICENSE.txt b/hudi-timeline-service/src/main/resources/META-INF/LICENSE.txt similarity index 100% rename from hoodie-timeline-service/src/main/resources/META-INF/LICENSE.txt rename to hudi-timeline-service/src/main/resources/META-INF/LICENSE.txt diff --git a/hoodie-timeline-service/src/main/resources/META-INF/NOTICE.txt b/hudi-timeline-service/src/main/resources/META-INF/NOTICE.txt similarity index 100% rename from hoodie-timeline-service/src/main/resources/META-INF/NOTICE.txt rename to hudi-timeline-service/src/main/resources/META-INF/NOTICE.txt diff --git a/hoodie-timeline-service/src/test/java/com/uber/hoodie/timeline/table/view/RemoteHoodieTableFileSystemViewTest.java b/hudi-timeline-service/src/test/java/org/apache/hudi/timeline/table/view/RemoteHoodieTableFileSystemViewTest.java similarity index 77% rename from hoodie-timeline-service/src/test/java/com/uber/hoodie/timeline/table/view/RemoteHoodieTableFileSystemViewTest.java rename to hudi-timeline-service/src/test/java/org/apache/hudi/timeline/table/view/RemoteHoodieTableFileSystemViewTest.java index 5d3d283ef..cf901e039 100644 --- a/hoodie-timeline-service/src/test/java/com/uber/hoodie/timeline/table/view/RemoteHoodieTableFileSystemViewTest.java +++ b/hudi-timeline-service/src/test/java/org/apache/hudi/timeline/table/view/RemoteHoodieTableFileSystemViewTest.java @@ -16,17 +16,17 @@ * limitations under the License. */ -package com.uber.hoodie.timeline.table.view; +package org.apache.hudi.timeline.table.view; -import com.uber.hoodie.common.SerializableConfiguration; -import com.uber.hoodie.common.table.HoodieTimeline; -import com.uber.hoodie.common.table.SyncableFileSystemView; -import com.uber.hoodie.common.table.view.FileSystemViewManager; -import com.uber.hoodie.common.table.view.FileSystemViewStorageConfig; -import com.uber.hoodie.common.table.view.FileSystemViewStorageType; -import com.uber.hoodie.common.table.view.HoodieTableFileSystemViewTest; -import com.uber.hoodie.common.table.view.RemoteHoodieTableFileSystemView; -import com.uber.hoodie.timeline.service.TimelineService; +import org.apache.hudi.common.SerializableConfiguration; +import org.apache.hudi.common.table.HoodieTimeline; +import org.apache.hudi.common.table.SyncableFileSystemView; +import org.apache.hudi.common.table.view.FileSystemViewManager; +import org.apache.hudi.common.table.view.FileSystemViewStorageConfig; +import org.apache.hudi.common.table.view.FileSystemViewStorageType; +import org.apache.hudi.common.table.view.HoodieTableFileSystemViewTest; +import org.apache.hudi.common.table.view.RemoteHoodieTableFileSystemView; +import org.apache.hudi.timeline.service.TimelineService; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; diff --git a/hoodie-timeline-service/src/test/resources/log4j-surefire-quiet.properties b/hudi-timeline-service/src/test/resources/log4j-surefire-quiet.properties similarity index 100% rename from hoodie-timeline-service/src/test/resources/log4j-surefire-quiet.properties rename to hudi-timeline-service/src/test/resources/log4j-surefire-quiet.properties diff --git a/hoodie-timeline-service/src/test/resources/log4j-surefire.properties b/hudi-timeline-service/src/test/resources/log4j-surefire.properties similarity index 100% rename from hoodie-timeline-service/src/test/resources/log4j-surefire.properties rename to hudi-timeline-service/src/test/resources/log4j-surefire.properties diff --git a/hoodie-utilities/pom.xml b/hudi-utilities/pom.xml similarity index 92% rename from hoodie-utilities/pom.xml rename to hudi-utilities/pom.xml index 30b61ff46..a571345b2 100644 --- a/hoodie-utilities/pom.xml +++ b/hudi-utilities/pom.xml @@ -17,13 +17,13 @@ --> - hoodie - com.uber.hoodie - 0.4.8-SNAPSHOT + hudi + org.apache.hudi + 0.5.0-SNAPSHOT 4.0.0 - hoodie-utilities + hudi-utilities jar @@ -99,14 +99,14 @@ - com.uber.hoodie - hoodie-common + org.apache.hudi + hudi-common ${project.version} - com.uber.hoodie - hoodie-common + org.apache.hudi + hudi-common ${project.version} tests test-jar @@ -114,8 +114,8 @@ - com.uber.hoodie - hoodie-hive + org.apache.hudi + hudi-hive ${project.version} tests test-jar @@ -123,8 +123,8 @@ - com.uber.hoodie - hoodie-spark + org.apache.hudi + hudi-spark ${project.version} @@ -195,8 +195,8 @@ - com.uber.hoodie - hoodie-hive + org.apache.hudi + hudi-hive ${project.version} @@ -207,14 +207,14 @@ - com.uber.hoodie - hoodie-client + org.apache.hudi + hudi-client ${project.version} - com.uber.hoodie - hoodie-client + org.apache.hudi + hudi-client ${project.version} tests test-jar diff --git a/hoodie-utilities/src/main/java/com/uber/hoodie/utilities/HDFSParquetImporter.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/HDFSParquetImporter.java similarity index 94% rename from hoodie-utilities/src/main/java/com/uber/hoodie/utilities/HDFSParquetImporter.java rename to hudi-utilities/src/main/java/org/apache/hudi/utilities/HDFSParquetImporter.java index b5ddb2220..6af7d3165 100644 --- a/hoodie-utilities/src/main/java/com/uber/hoodie/utilities/HDFSParquetImporter.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/HDFSParquetImporter.java @@ -16,25 +16,13 @@ * limitations under the License. */ -package com.uber.hoodie.utilities; +package org.apache.hudi.utilities; import com.beust.jcommander.IValueValidator; import com.beust.jcommander.JCommander; import com.beust.jcommander.Parameter; import com.beust.jcommander.ParameterException; import com.google.common.annotations.VisibleForTesting; -import com.uber.hoodie.HoodieWriteClient; -import com.uber.hoodie.WriteStatus; -import com.uber.hoodie.common.HoodieJsonPayload; -import com.uber.hoodie.common.model.HoodieKey; -import com.uber.hoodie.common.model.HoodieRecord; -import com.uber.hoodie.common.model.HoodieRecordPayload; -import com.uber.hoodie.common.table.HoodieTableConfig; -import com.uber.hoodie.common.table.HoodieTableMetaClient; -import com.uber.hoodie.common.util.FSUtils; -import com.uber.hoodie.common.util.Option; -import com.uber.hoodie.common.util.TypedProperties; -import com.uber.hoodie.exception.HoodieIOException; import java.io.IOException; import java.io.Serializable; import java.text.SimpleDateFormat; @@ -49,6 +37,18 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.mapreduce.Job; import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; +import org.apache.hudi.HoodieWriteClient; +import org.apache.hudi.WriteStatus; +import org.apache.hudi.common.HoodieJsonPayload; +import org.apache.hudi.common.model.HoodieKey; +import org.apache.hudi.common.model.HoodieRecord; +import org.apache.hudi.common.model.HoodieRecordPayload; +import org.apache.hudi.common.table.HoodieTableConfig; +import org.apache.hudi.common.table.HoodieTableMetaClient; +import org.apache.hudi.common.util.FSUtils; +import org.apache.hudi.common.util.Option; +import org.apache.hudi.common.util.TypedProperties; +import org.apache.hudi.exception.HoodieIOException; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; import org.apache.parquet.avro.AvroReadSupport; diff --git a/hoodie-utilities/src/main/java/com/uber/hoodie/utilities/HiveIncrementalPuller.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/HiveIncrementalPuller.java similarity index 97% rename from hoodie-utilities/src/main/java/com/uber/hoodie/utilities/HiveIncrementalPuller.java rename to hudi-utilities/src/main/java/org/apache/hudi/utilities/HiveIncrementalPuller.java index 86788939c..5f45a1898 100644 --- a/hoodie-utilities/src/main/java/com/uber/hoodie/utilities/HiveIncrementalPuller.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/HiveIncrementalPuller.java @@ -16,16 +16,10 @@ * limitations under the License. */ -package com.uber.hoodie.utilities; +package org.apache.hudi.utilities; import com.beust.jcommander.JCommander; import com.beust.jcommander.Parameter; -import com.uber.hoodie.common.table.HoodieTableMetaClient; -import com.uber.hoodie.common.table.timeline.HoodieInstant; -import com.uber.hoodie.common.util.Option; -import com.uber.hoodie.exception.HoodieException; -import com.uber.hoodie.utilities.exception.HoodieIncrementalPullException; -import com.uber.hoodie.utilities.exception.HoodieIncrementalPullSQLException; import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; @@ -45,6 +39,12 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.permission.FsAction; import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hudi.common.table.HoodieTableMetaClient; +import org.apache.hudi.common.table.timeline.HoodieInstant; +import org.apache.hudi.common.util.Option; +import org.apache.hudi.exception.HoodieException; +import org.apache.hudi.utilities.exception.HoodieIncrementalPullException; +import org.apache.hudi.utilities.exception.HoodieIncrementalPullSQLException; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; import org.stringtemplate.v4.ST; @@ -214,7 +214,7 @@ public class HiveIncrementalPuller { executeStatement("set mapred.job.queue.name=" + config.yarnQueueName, stmt); // Set the inputformat to HoodieCombineHiveInputFormat executeStatement( - "set hive.input.format=com.uber.hoodie.hadoop.hive.HoodieCombineHiveInputFormat", stmt); + "set hive.input.format=org.apache.hudi.hadoop.hive.HoodieCombineHiveInputFormat", stmt); // Allow queries without partition predicate executeStatement("set hive.strict.checks.large.query=false", stmt); // Dont gather stats for the table created diff --git a/hoodie-utilities/src/main/java/com/uber/hoodie/utilities/HoodieCleaner.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieCleaner.java similarity index 94% rename from hoodie-utilities/src/main/java/com/uber/hoodie/utilities/HoodieCleaner.java rename to hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieCleaner.java index a6cd03dcb..aee1ca436 100644 --- a/hoodie-utilities/src/main/java/com/uber/hoodie/utilities/HoodieCleaner.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieCleaner.java @@ -16,20 +16,20 @@ * limitations under the License. */ -package com.uber.hoodie.utilities; +package org.apache.hudi.utilities; import com.beust.jcommander.JCommander; import com.beust.jcommander.Parameter; -import com.uber.hoodie.HoodieWriteClient; -import com.uber.hoodie.common.util.FSUtils; -import com.uber.hoodie.common.util.TypedProperties; -import com.uber.hoodie.config.HoodieWriteConfig; import java.io.IOException; import java.io.Serializable; import java.util.ArrayList; import java.util.List; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hudi.HoodieWriteClient; +import org.apache.hudi.common.util.FSUtils; +import org.apache.hudi.common.util.TypedProperties; +import org.apache.hudi.config.HoodieWriteConfig; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; import org.apache.spark.api.java.JavaSparkContext; diff --git a/hoodie-utilities/src/main/java/com/uber/hoodie/utilities/HoodieCompactionAdminTool.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieCompactionAdminTool.java similarity index 94% rename from hoodie-utilities/src/main/java/com/uber/hoodie/utilities/HoodieCompactionAdminTool.java rename to hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieCompactionAdminTool.java index 0c640f6d5..74a4128f3 100644 --- a/hoodie-utilities/src/main/java/com/uber/hoodie/utilities/HoodieCompactionAdminTool.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieCompactionAdminTool.java @@ -16,22 +16,22 @@ * limitations under the License. */ -package com.uber.hoodie.utilities; +package org.apache.hudi.utilities; import com.beust.jcommander.JCommander; import com.beust.jcommander.Parameter; -import com.uber.hoodie.CompactionAdminClient; -import com.uber.hoodie.CompactionAdminClient.RenameOpResult; -import com.uber.hoodie.CompactionAdminClient.ValidationOpResult; -import com.uber.hoodie.common.model.HoodieFileGroupId; -import com.uber.hoodie.common.table.HoodieTableMetaClient; -import com.uber.hoodie.common.util.FSUtils; import java.io.ObjectOutputStream; import java.io.Serializable; import java.util.List; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hudi.CompactionAdminClient; +import org.apache.hudi.CompactionAdminClient.RenameOpResult; +import org.apache.hudi.CompactionAdminClient.ValidationOpResult; +import org.apache.hudi.common.model.HoodieFileGroupId; +import org.apache.hudi.common.table.HoodieTableMetaClient; +import org.apache.hudi.common.util.FSUtils; import org.apache.spark.api.java.JavaSparkContext; public class HoodieCompactionAdminTool { diff --git a/hoodie-utilities/src/main/java/com/uber/hoodie/utilities/HoodieCompactor.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieCompactor.java similarity index 95% rename from hoodie-utilities/src/main/java/com/uber/hoodie/utilities/HoodieCompactor.java rename to hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieCompactor.java index 0e3657118..7aa56926b 100644 --- a/hoodie-utilities/src/main/java/com/uber/hoodie/utilities/HoodieCompactor.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieCompactor.java @@ -16,20 +16,20 @@ * limitations under the License. */ -package com.uber.hoodie.utilities; +package org.apache.hudi.utilities; import com.beust.jcommander.JCommander; import com.beust.jcommander.Parameter; -import com.uber.hoodie.HoodieWriteClient; -import com.uber.hoodie.WriteStatus; -import com.uber.hoodie.common.util.FSUtils; -import com.uber.hoodie.common.util.Option; -import com.uber.hoodie.common.util.TypedProperties; import java.io.Serializable; import java.util.ArrayList; import java.util.List; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hudi.HoodieWriteClient; +import org.apache.hudi.WriteStatus; +import org.apache.hudi.common.util.FSUtils; +import org.apache.hudi.common.util.Option; +import org.apache.hudi.common.util.TypedProperties; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; import org.apache.spark.api.java.JavaRDD; diff --git a/hoodie-utilities/src/main/java/com/uber/hoodie/utilities/HoodieSnapshotCopier.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieSnapshotCopier.java similarity index 91% rename from hoodie-utilities/src/main/java/com/uber/hoodie/utilities/HoodieSnapshotCopier.java rename to hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieSnapshotCopier.java index 337e1ca2c..2a7054893 100644 --- a/hoodie-utilities/src/main/java/com/uber/hoodie/utilities/HoodieSnapshotCopier.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieSnapshotCopier.java @@ -16,21 +16,10 @@ * limitations under the License. */ -package com.uber.hoodie.utilities; +package org.apache.hudi.utilities; import com.beust.jcommander.JCommander; import com.beust.jcommander.Parameter; -import com.uber.hoodie.common.SerializableConfiguration; -import com.uber.hoodie.common.model.HoodieDataFile; -import com.uber.hoodie.common.model.HoodiePartitionMetadata; -import com.uber.hoodie.common.table.HoodieTableConfig; -import com.uber.hoodie.common.table.HoodieTableMetaClient; -import com.uber.hoodie.common.table.HoodieTimeline; -import com.uber.hoodie.common.table.TableFileSystemView; -import com.uber.hoodie.common.table.timeline.HoodieInstant; -import com.uber.hoodie.common.table.view.HoodieTableFileSystemView; -import com.uber.hoodie.common.util.FSUtils; -import com.uber.hoodie.common.util.Option; import java.io.IOException; import java.io.Serializable; import java.util.ArrayList; @@ -40,6 +29,17 @@ import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.fs.Path; +import org.apache.hudi.common.SerializableConfiguration; +import org.apache.hudi.common.model.HoodieDataFile; +import org.apache.hudi.common.model.HoodiePartitionMetadata; +import org.apache.hudi.common.table.HoodieTableConfig; +import org.apache.hudi.common.table.HoodieTableMetaClient; +import org.apache.hudi.common.table.HoodieTimeline; +import org.apache.hudi.common.table.TableFileSystemView.ReadOptimizedView; +import org.apache.hudi.common.table.timeline.HoodieInstant; +import org.apache.hudi.common.table.view.HoodieTableFileSystemView; +import org.apache.hudi.common.util.FSUtils; +import org.apache.hudi.common.util.Option; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; import org.apache.spark.SparkConf; @@ -75,7 +75,7 @@ public class HoodieSnapshotCopier implements Serializable { final SerializableConfiguration serConf = new SerializableConfiguration( jsc.hadoopConfiguration()); final HoodieTableMetaClient tableMetadata = new HoodieTableMetaClient(fs.getConf(), baseDir); - final TableFileSystemView.ReadOptimizedView fsView = new HoodieTableFileSystemView( + final ReadOptimizedView fsView = new HoodieTableFileSystemView( tableMetadata, tableMetadata.getActiveTimeline().getCommitsTimeline().filterCompletedInstants()); // Get the latest commit diff --git a/hoodie-utilities/src/main/java/com/uber/hoodie/utilities/HoodieWithTimelineServer.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieWithTimelineServer.java similarity index 99% rename from hoodie-utilities/src/main/java/com/uber/hoodie/utilities/HoodieWithTimelineServer.java rename to hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieWithTimelineServer.java index 0aa7f8e93..dfab9b3c1 100644 --- a/hoodie-utilities/src/main/java/com/uber/hoodie/utilities/HoodieWithTimelineServer.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieWithTimelineServer.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package com.uber.hoodie.utilities; +package org.apache.hudi.utilities; import com.beust.jcommander.JCommander; import com.beust.jcommander.Parameter; diff --git a/hoodie-utilities/src/main/java/com/uber/hoodie/utilities/UtilHelpers.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/UtilHelpers.java similarity index 92% rename from hoodie-utilities/src/main/java/com/uber/hoodie/utilities/UtilHelpers.java rename to hudi-utilities/src/main/java/org/apache/hudi/utilities/UtilHelpers.java index 24ff8f599..36a9bd790 100644 --- a/hoodie-utilities/src/main/java/com/uber/hoodie/utilities/UtilHelpers.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/UtilHelpers.java @@ -16,23 +16,9 @@ * limitations under the License. */ -package com.uber.hoodie.utilities; +package org.apache.hudi.utilities; import com.google.common.base.Preconditions; -import com.uber.hoodie.HoodieWriteClient; -import com.uber.hoodie.WriteStatus; -import com.uber.hoodie.common.util.DFSPropertiesConfiguration; -import com.uber.hoodie.common.util.Option; -import com.uber.hoodie.common.util.ReflectionUtils; -import com.uber.hoodie.common.util.TypedProperties; -import com.uber.hoodie.config.HoodieCompactionConfig; -import com.uber.hoodie.config.HoodieIndexConfig; -import com.uber.hoodie.config.HoodieWriteConfig; -import com.uber.hoodie.exception.HoodieException; -import com.uber.hoodie.index.HoodieIndex; -import com.uber.hoodie.utilities.schema.SchemaProvider; -import com.uber.hoodie.utilities.sources.Source; -import com.uber.hoodie.utilities.transform.Transformer; import java.io.BufferedReader; import java.io.IOException; import java.io.InputStream; @@ -44,6 +30,20 @@ import java.util.Map; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hudi.HoodieWriteClient; +import org.apache.hudi.WriteStatus; +import org.apache.hudi.common.util.DFSPropertiesConfiguration; +import org.apache.hudi.common.util.Option; +import org.apache.hudi.common.util.ReflectionUtils; +import org.apache.hudi.common.util.TypedProperties; +import org.apache.hudi.config.HoodieCompactionConfig; +import org.apache.hudi.config.HoodieIndexConfig; +import org.apache.hudi.config.HoodieWriteConfig; +import org.apache.hudi.exception.HoodieException; +import org.apache.hudi.index.HoodieIndex; +import org.apache.hudi.utilities.schema.SchemaProvider; +import org.apache.hudi.utilities.sources.Source; +import org.apache.hudi.utilities.transform.Transformer; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; import org.apache.spark.Accumulator; diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/adhoc/UpgradePayloadFromUberToApache.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/adhoc/UpgradePayloadFromUberToApache.java new file mode 100644 index 000000000..7386af880 --- /dev/null +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/adhoc/UpgradePayloadFromUberToApache.java @@ -0,0 +1,114 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.utilities.adhoc; + +import com.beust.jcommander.JCommander; +import com.beust.jcommander.Parameter; +import java.io.BufferedReader; +import java.io.FileReader; +import java.io.IOException; +import java.io.Serializable; +import java.util.HashMap; +import java.util.Map; +import java.util.Properties; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.hudi.common.model.HoodieTableType; +import org.apache.hudi.common.table.HoodieTableConfig; +import org.apache.hudi.common.table.HoodieTableMetaClient; +import org.apache.hudi.common.util.FSUtils; +import org.apache.hudi.config.HoodieCompactionConfig; +import org.apache.log4j.LogManager; +import org.apache.log4j.Logger; + +/** + * This is an one-time use class meant for migrating the configuration for + * "hoodie.compaction.payload.class" in .hoodie/hoodie.properties from com.uber.hoodie to + * org.apache.hudi + * It takes in a file containing base-paths for a set of hudi datasets and does the migration + */ +public class UpgradePayloadFromUberToApache implements Serializable { + + private static Logger logger = LogManager.getLogger(UpgradePayloadFromUberToApache.class); + + private final Config cfg; + + public UpgradePayloadFromUberToApache(Config cfg) { + this.cfg = cfg; + } + + public void run() throws IOException { + BufferedReader reader = new BufferedReader(new FileReader(cfg.inputPath)); + String basePath = reader.readLine(); + while (basePath != null) { + basePath = basePath.trim(); + if (!basePath.startsWith("#")) { + logger.info("Performing upgrade for " + basePath); + String metaPath = String.format("%s/.hoodie", basePath); + HoodieTableMetaClient metaClient = new HoodieTableMetaClient( + FSUtils.prepareHadoopConf(new Configuration()), basePath, false); + HoodieTableConfig tableConfig = metaClient.getTableConfig(); + if (tableConfig.getTableType().equals(HoodieTableType.MERGE_ON_READ)) { + Map propsMap = tableConfig.getProps(); + if (propsMap.containsKey(HoodieCompactionConfig.PAYLOAD_CLASS_PROP)) { + String payloadClass = propsMap.get(HoodieCompactionConfig.PAYLOAD_CLASS_PROP); + logger.info("Found payload class=" + payloadClass); + if (payloadClass.startsWith("com.uber.hoodie")) { + String newPayloadClass = payloadClass.replace("com.uber.hoodie", + "org.apache.hudi"); + logger.info("Replacing payload class (" + payloadClass + + ") with (" + newPayloadClass + ")"); + Map newPropsMap = new HashMap<>(propsMap); + newPropsMap.put(HoodieCompactionConfig.PAYLOAD_CLASS_PROP, newPayloadClass); + Properties props = new Properties(); + props.putAll(newPropsMap); + tableConfig.createHoodieProperties(metaClient.getFs(), new Path(metaPath), props); + logger.info("Finished upgrade for " + basePath); + } + } + } else { + logger.info("Skipping as this table is COW table. BasePath=" + basePath); + + } + } + } + } + + public static class Config implements Serializable { + + @Parameter(names = {"--datasets_list_path", "-sp"}, + description = "Local File containing list of base-paths for which migration needs to be performed", + required = true) + public String inputPath = null; + @Parameter(names = {"--help", "-h"}, help = true) + public Boolean help = false; + } + + public static void main(String[] args) throws Exception { + final Config cfg = new Config(); + JCommander cmd = new JCommander(cfg, args); + if (cfg.help || args.length == 0) { + cmd.usage(); + System.exit(1); + } + UpgradePayloadFromUberToApache upgrader = new UpgradePayloadFromUberToApache(cfg); + upgrader.run(); + } + +} diff --git a/hoodie-utilities/src/main/java/com/uber/hoodie/utilities/deltastreamer/AbstractDeltaStreamerService.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/deltastreamer/AbstractDeltaStreamerService.java similarity index 97% rename from hoodie-utilities/src/main/java/com/uber/hoodie/utilities/deltastreamer/AbstractDeltaStreamerService.java rename to hudi-utilities/src/main/java/org/apache/hudi/utilities/deltastreamer/AbstractDeltaStreamerService.java index 179fdbd28..e3d900bbb 100644 --- a/hoodie-utilities/src/main/java/com/uber/hoodie/utilities/deltastreamer/AbstractDeltaStreamerService.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/deltastreamer/AbstractDeltaStreamerService.java @@ -16,9 +16,8 @@ * limitations under the License. */ -package com.uber.hoodie.utilities.deltastreamer; +package org.apache.hudi.utilities.deltastreamer; -import com.uber.hoodie.common.util.collection.Pair; import java.io.Serializable; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; @@ -26,6 +25,7 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; import java.util.function.Function; +import org.apache.hudi.common.util.collection.Pair; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; diff --git a/hoodie-utilities/src/main/java/com/uber/hoodie/utilities/deltastreamer/Compactor.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/deltastreamer/Compactor.java similarity index 88% rename from hoodie-utilities/src/main/java/com/uber/hoodie/utilities/deltastreamer/Compactor.java rename to hudi-utilities/src/main/java/org/apache/hudi/utilities/deltastreamer/Compactor.java index d07b5e517..7a2a3f672 100644 --- a/hoodie-utilities/src/main/java/com/uber/hoodie/utilities/deltastreamer/Compactor.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/deltastreamer/Compactor.java @@ -16,15 +16,15 @@ * limitations under the License. */ -package com.uber.hoodie.utilities.deltastreamer; +package org.apache.hudi.utilities.deltastreamer; -import com.uber.hoodie.HoodieWriteClient; -import com.uber.hoodie.WriteStatus; -import com.uber.hoodie.common.table.timeline.HoodieInstant; -import com.uber.hoodie.common.util.Option; -import com.uber.hoodie.exception.HoodieException; import java.io.IOException; import java.io.Serializable; +import org.apache.hudi.HoodieWriteClient; +import org.apache.hudi.WriteStatus; +import org.apache.hudi.common.table.timeline.HoodieInstant; +import org.apache.hudi.common.util.Option; +import org.apache.hudi.exception.HoodieException; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; import org.apache.spark.api.java.JavaRDD; diff --git a/hoodie-utilities/src/main/java/com/uber/hoodie/utilities/deltastreamer/DeltaSync.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/deltastreamer/DeltaSync.java similarity index 91% rename from hoodie-utilities/src/main/java/com/uber/hoodie/utilities/deltastreamer/DeltaSync.java rename to hudi-utilities/src/main/java/org/apache/hudi/utilities/deltastreamer/DeltaSync.java index 33bb6bfa8..075e1c954 100644 --- a/hoodie-utilities/src/main/java/com/uber/hoodie/utilities/deltastreamer/DeltaSync.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/deltastreamer/DeltaSync.java @@ -16,41 +16,12 @@ * limitations under the License. */ -package com.uber.hoodie.utilities.deltastreamer; +package org.apache.hudi.utilities.deltastreamer; -import static com.uber.hoodie.utilities.schema.RowBasedSchemaProvider.HOODIE_RECORD_NAMESPACE; -import static com.uber.hoodie.utilities.schema.RowBasedSchemaProvider.HOODIE_RECORD_STRUCT_NAME; +import static org.apache.hudi.utilities.schema.RowBasedSchemaProvider.HOODIE_RECORD_NAMESPACE; +import static org.apache.hudi.utilities.schema.RowBasedSchemaProvider.HOODIE_RECORD_STRUCT_NAME; import com.codahale.metrics.Timer; -import com.uber.hoodie.AvroConversionUtils; -import com.uber.hoodie.DataSourceUtils; -import com.uber.hoodie.HoodieWriteClient; -import com.uber.hoodie.KeyGenerator; -import com.uber.hoodie.WriteStatus; -import com.uber.hoodie.common.model.HoodieCommitMetadata; -import com.uber.hoodie.common.model.HoodieRecord; -import com.uber.hoodie.common.model.HoodieRecordPayload; -import com.uber.hoodie.common.model.HoodieTableType; -import com.uber.hoodie.common.table.HoodieTableMetaClient; -import com.uber.hoodie.common.table.HoodieTimeline; -import com.uber.hoodie.common.table.timeline.HoodieInstant; -import com.uber.hoodie.common.util.Option; -import com.uber.hoodie.common.util.TypedProperties; -import com.uber.hoodie.common.util.collection.Pair; -import com.uber.hoodie.config.HoodieCompactionConfig; -import com.uber.hoodie.config.HoodieIndexConfig; -import com.uber.hoodie.config.HoodieWriteConfig; -import com.uber.hoodie.exception.HoodieException; -import com.uber.hoodie.hive.HiveSyncConfig; -import com.uber.hoodie.hive.HiveSyncTool; -import com.uber.hoodie.index.HoodieIndex; -import com.uber.hoodie.utilities.UtilHelpers; -import com.uber.hoodie.utilities.deltastreamer.HoodieDeltaStreamer.Operation; -import com.uber.hoodie.utilities.exception.HoodieDeltaStreamerException; -import com.uber.hoodie.utilities.schema.RowBasedSchemaProvider; -import com.uber.hoodie.utilities.schema.SchemaProvider; -import com.uber.hoodie.utilities.sources.InputBatch; -import com.uber.hoodie.utilities.transform.Transformer; import java.io.IOException; import java.io.Serializable; import java.util.Arrays; @@ -63,6 +34,35 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hudi.AvroConversionUtils; +import org.apache.hudi.DataSourceUtils; +import org.apache.hudi.HoodieWriteClient; +import org.apache.hudi.KeyGenerator; +import org.apache.hudi.WriteStatus; +import org.apache.hudi.common.model.HoodieCommitMetadata; +import org.apache.hudi.common.model.HoodieRecord; +import org.apache.hudi.common.model.HoodieRecordPayload; +import org.apache.hudi.common.model.HoodieTableType; +import org.apache.hudi.common.table.HoodieTableMetaClient; +import org.apache.hudi.common.table.HoodieTimeline; +import org.apache.hudi.common.table.timeline.HoodieInstant; +import org.apache.hudi.common.util.Option; +import org.apache.hudi.common.util.TypedProperties; +import org.apache.hudi.common.util.collection.Pair; +import org.apache.hudi.config.HoodieCompactionConfig; +import org.apache.hudi.config.HoodieIndexConfig; +import org.apache.hudi.config.HoodieWriteConfig; +import org.apache.hudi.exception.HoodieException; +import org.apache.hudi.hive.HiveSyncConfig; +import org.apache.hudi.hive.HiveSyncTool; +import org.apache.hudi.index.HoodieIndex; +import org.apache.hudi.utilities.UtilHelpers; +import org.apache.hudi.utilities.deltastreamer.HoodieDeltaStreamer.Operation; +import org.apache.hudi.utilities.exception.HoodieDeltaStreamerException; +import org.apache.hudi.utilities.schema.RowBasedSchemaProvider; +import org.apache.hudi.utilities.schema.SchemaProvider; +import org.apache.hudi.utilities.sources.InputBatch; +import org.apache.hudi.utilities.transform.Transformer; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; import org.apache.spark.api.java.JavaRDD; @@ -429,7 +429,7 @@ public class DeltaSync implements Serializable { /** * Sync to Hive */ - private void syncHive() { + private void syncHive() throws ClassNotFoundException { if (cfg.enableHiveSync) { HiveSyncConfig hiveSyncConfig = DataSourceUtils.buildHiveSyncConfig(props, cfg.targetBasePath); log.info("Syncing target hoodie table with hive table(" + hiveSyncConfig.tableName diff --git a/hoodie-utilities/src/main/java/com/uber/hoodie/utilities/deltastreamer/HoodieDeltaStreamer.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/deltastreamer/HoodieDeltaStreamer.java similarity index 93% rename from hoodie-utilities/src/main/java/com/uber/hoodie/utilities/deltastreamer/HoodieDeltaStreamer.java rename to hudi-utilities/src/main/java/org/apache/hudi/utilities/deltastreamer/HoodieDeltaStreamer.java index 5de5778f6..cff6a8bfd 100644 --- a/hoodie-utilities/src/main/java/com/uber/hoodie/utilities/deltastreamer/HoodieDeltaStreamer.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/deltastreamer/HoodieDeltaStreamer.java @@ -16,32 +16,13 @@ * limitations under the License. */ -package com.uber.hoodie.utilities.deltastreamer; - -import static com.uber.hoodie.common.table.HoodieTimeline.COMPACTION_ACTION; +package org.apache.hudi.utilities.deltastreamer; import com.beust.jcommander.IStringConverter; import com.beust.jcommander.JCommander; import com.beust.jcommander.Parameter; import com.beust.jcommander.ParameterException; import com.google.common.base.Preconditions; -import com.uber.hoodie.HoodieWriteClient; -import com.uber.hoodie.OverwriteWithLatestAvroPayload; -import com.uber.hoodie.common.model.HoodieTableType; -import com.uber.hoodie.common.table.HoodieTableMetaClient; -import com.uber.hoodie.common.table.timeline.HoodieInstant; -import com.uber.hoodie.common.table.timeline.HoodieInstant.State; -import com.uber.hoodie.common.util.CompactionUtils; -import com.uber.hoodie.common.util.FSUtils; -import com.uber.hoodie.common.util.Option; -import com.uber.hoodie.common.util.TypedProperties; -import com.uber.hoodie.common.util.collection.Pair; -import com.uber.hoodie.exception.HoodieException; -import com.uber.hoodie.exception.HoodieIOException; -import com.uber.hoodie.utilities.HiveIncrementalPuller; -import com.uber.hoodie.utilities.UtilHelpers; -import com.uber.hoodie.utilities.schema.SchemaProvider; -import com.uber.hoodie.utilities.sources.JsonDFSSource; import java.io.IOException; import java.io.Serializable; import java.util.ArrayList; @@ -61,6 +42,24 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hudi.HoodieWriteClient; +import org.apache.hudi.OverwriteWithLatestAvroPayload; +import org.apache.hudi.common.model.HoodieTableType; +import org.apache.hudi.common.table.HoodieTableMetaClient; +import org.apache.hudi.common.table.HoodieTimeline; +import org.apache.hudi.common.table.timeline.HoodieInstant; +import org.apache.hudi.common.table.timeline.HoodieInstant.State; +import org.apache.hudi.common.util.CompactionUtils; +import org.apache.hudi.common.util.FSUtils; +import org.apache.hudi.common.util.Option; +import org.apache.hudi.common.util.TypedProperties; +import org.apache.hudi.common.util.collection.Pair; +import org.apache.hudi.exception.HoodieException; +import org.apache.hudi.exception.HoodieIOException; +import org.apache.hudi.utilities.HiveIncrementalPuller; +import org.apache.hudi.utilities.UtilHelpers; +import org.apache.hudi.utilities.schema.SchemaProvider; +import org.apache.hudi.utilities.sources.JsonDFSSource; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; import org.apache.spark.api.java.JavaSparkContext; @@ -167,12 +166,12 @@ public class HoodieDeltaStreamer implements Serializable { public String propsFilePath = "file://" + System.getProperty("user.dir") + "/src/test/resources/delta-streamer-config/dfs-source.properties"; - @Parameter(names = {"--hoodie-conf"}, description = "Any configuration that can be set in the properties file " + @Parameter(names = {"--hudi-conf"}, description = "Any configuration that can be set in the properties file " + "(using the CLI parameter \"--propsFilePath\") can also be passed command line using this parameter") public List configs = new ArrayList<>(); - @Parameter(names = {"--source-class"}, description = "Subclass of com.uber.hoodie.utilities.sources to read data. " - + "Built-in options: com.uber.hoodie.utilities.sources.{JsonDFSSource (default), AvroDFSSource, " + @Parameter(names = {"--source-class"}, description = "Subclass of org.apache.hudi.utilities.sources to read data. " + + "Built-in options: org.apache.hudi.utilities.sources.{JsonDFSSource (default), AvroDFSSource, " + "JsonKafkaSource, AvroKafkaSource, HiveIncrPullSource}") public String sourceClassName = JsonDFSSource.class.getName(); @@ -184,18 +183,18 @@ public class HoodieDeltaStreamer implements Serializable { + "a GenericRecord. Implement your own, if you want to do something other than overwriting existing value") public String payloadClassName = OverwriteWithLatestAvroPayload.class.getName(); - @Parameter(names = {"--schemaprovider-class"}, description = "subclass of com.uber.hoodie.utilities.schema" + @Parameter(names = {"--schemaprovider-class"}, description = "subclass of org.apache.hudi.utilities.schema" + ".SchemaProvider to attach schemas to input & target table data, built in options: " - + "com.uber.hoodie.utilities.schema.FilebasedSchemaProvider." - + "Source (See com.uber.hoodie.utilities.sources.Source) implementation can implement their own SchemaProvider." + + "org.apache.hudi.utilities.schema.FilebasedSchemaProvider." + + "Source (See org.apache.hudi.utilities.sources.Source) implementation can implement their own SchemaProvider." + " For Sources that return Dataset, the schema is obtained implicitly. " + "However, this CLI option allows overriding the schemaprovider returned by Source.") public String schemaProviderClassName = null; @Parameter(names = {"--transformer-class"}, - description = "subclass of com.uber.hoodie.utilities.transform.Transformer" + description = "subclass of org.apache.hudi.utilities.transform.Transformer" + ". Allows transforming raw source dataset to a target dataset (conforming to target schema) before " - + "writing. Default : Not set. E:g - com.uber.hoodie.utilities.transform.SqlQueryBasedTransformer (which " + + "writing. Default : Not set. E:g - org.apache.hudi.utilities.transform.SqlQueryBasedTransformer (which " + "allows a SQL query templated to be passed as a transformation function)") public String transformerClassName = null; @@ -388,7 +387,8 @@ public class HoodieDeltaStreamer implements Serializable { Option scheduledCompactionInstant = deltaSync.syncOnce(); if (scheduledCompactionInstant.isPresent()) { log.info("Enqueuing new pending compaction instant (" + scheduledCompactionInstant + ")"); - asyncCompactService.enqueuePendingCompaction(new HoodieInstant(State.REQUESTED, COMPACTION_ACTION, + asyncCompactService.enqueuePendingCompaction(new HoodieInstant( + State.REQUESTED, HoodieTimeline.COMPACTION_ACTION, scheduledCompactionInstant.get())); asyncCompactService.waitTillPendingCompactionsReducesTo(cfg.maxPendingCompactions); } diff --git a/hoodie-utilities/src/main/java/com/uber/hoodie/utilities/deltastreamer/HoodieDeltaStreamerMetrics.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/deltastreamer/HoodieDeltaStreamerMetrics.java similarity index 85% rename from hoodie-utilities/src/main/java/com/uber/hoodie/utilities/deltastreamer/HoodieDeltaStreamerMetrics.java rename to hudi-utilities/src/main/java/org/apache/hudi/utilities/deltastreamer/HoodieDeltaStreamerMetrics.java index d2eb5dd9f..4132f5b87 100644 --- a/hoodie-utilities/src/main/java/com/uber/hoodie/utilities/deltastreamer/HoodieDeltaStreamerMetrics.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/deltastreamer/HoodieDeltaStreamerMetrics.java @@ -16,13 +16,11 @@ * limitations under the License. */ -package com.uber.hoodie.utilities.deltastreamer; - -import static com.uber.hoodie.metrics.Metrics.registerGauge; +package org.apache.hudi.utilities.deltastreamer; import com.codahale.metrics.Timer; -import com.uber.hoodie.config.HoodieWriteConfig; -import com.uber.hoodie.metrics.Metrics; +import org.apache.hudi.config.HoodieWriteConfig; +import org.apache.hudi.metrics.Metrics; public class HoodieDeltaStreamerMetrics { @@ -68,8 +66,8 @@ public class HoodieDeltaStreamerMetrics { public void updateDeltaStreamerMetrics(long durationInNs, long hiveSyncNs) { if (config.isMetricsOn()) { - registerGauge(getMetricsName("deltastreamer", "duration"), getDurationInMs(durationInNs)); - registerGauge(getMetricsName("deltastreamer", "hiveSyncDuration"), getDurationInMs(hiveSyncNs)); + Metrics.registerGauge(getMetricsName("deltastreamer", "duration"), getDurationInMs(durationInNs)); + Metrics.registerGauge(getMetricsName("deltastreamer", "hiveSyncDuration"), getDurationInMs(hiveSyncNs)); } } diff --git a/hoodie-utilities/src/main/java/com/uber/hoodie/utilities/deltastreamer/SchedulerConfGenerator.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/deltastreamer/SchedulerConfGenerator.java similarity index 97% rename from hoodie-utilities/src/main/java/com/uber/hoodie/utilities/deltastreamer/SchedulerConfGenerator.java rename to hudi-utilities/src/main/java/org/apache/hudi/utilities/deltastreamer/SchedulerConfGenerator.java index 7ba6afff1..418df2763 100644 --- a/hoodie-utilities/src/main/java/com/uber/hoodie/utilities/deltastreamer/SchedulerConfGenerator.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/deltastreamer/SchedulerConfGenerator.java @@ -15,10 +15,8 @@ * */ -package com.uber.hoodie.utilities.deltastreamer; +package org.apache.hudi.utilities.deltastreamer; -import com.uber.hoodie.common.model.HoodieTableType; -import com.uber.hoodie.common.util.Option; import java.io.BufferedWriter; import java.io.File; import java.io.FileWriter; @@ -27,6 +25,8 @@ import java.util.HashMap; import java.util.Map; import java.util.UUID; import org.apache.commons.lang.text.StrSubstitutor; +import org.apache.hudi.common.model.HoodieTableType; +import org.apache.hudi.common.util.Option; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; import org.apache.spark.SparkConf; diff --git a/hoodie-utilities/src/main/java/com/uber/hoodie/utilities/deltastreamer/SourceFormatAdapter.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/deltastreamer/SourceFormatAdapter.java similarity index 88% rename from hoodie-utilities/src/main/java/com/uber/hoodie/utilities/deltastreamer/SourceFormatAdapter.java rename to hudi-utilities/src/main/java/org/apache/hudi/utilities/deltastreamer/SourceFormatAdapter.java index fc6901bdc..1b46d4afd 100644 --- a/hoodie-utilities/src/main/java/com/uber/hoodie/utilities/deltastreamer/SourceFormatAdapter.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/deltastreamer/SourceFormatAdapter.java @@ -16,21 +16,21 @@ * limitations under the License. */ -package com.uber.hoodie.utilities.deltastreamer; +package org.apache.hudi.utilities.deltastreamer; -import static com.uber.hoodie.utilities.schema.RowBasedSchemaProvider.HOODIE_RECORD_NAMESPACE; -import static com.uber.hoodie.utilities.schema.RowBasedSchemaProvider.HOODIE_RECORD_STRUCT_NAME; +import static org.apache.hudi.utilities.schema.RowBasedSchemaProvider.HOODIE_RECORD_NAMESPACE; +import static org.apache.hudi.utilities.schema.RowBasedSchemaProvider.HOODIE_RECORD_STRUCT_NAME; -import com.uber.hoodie.AvroConversionUtils; -import com.uber.hoodie.common.util.Option; -import com.uber.hoodie.utilities.sources.AvroSource; -import com.uber.hoodie.utilities.sources.InputBatch; -import com.uber.hoodie.utilities.sources.JsonSource; -import com.uber.hoodie.utilities.sources.RowSource; -import com.uber.hoodie.utilities.sources.Source; -import com.uber.hoodie.utilities.sources.helpers.AvroConvertor; import org.apache.avro.Schema; import org.apache.avro.generic.GenericRecord; +import org.apache.hudi.AvroConversionUtils; +import org.apache.hudi.common.util.Option; +import org.apache.hudi.utilities.sources.AvroSource; +import org.apache.hudi.utilities.sources.InputBatch; +import org.apache.hudi.utilities.sources.JsonSource; +import org.apache.hudi.utilities.sources.RowSource; +import org.apache.hudi.utilities.sources.Source; +import org.apache.hudi.utilities.sources.helpers.AvroConvertor; import org.apache.spark.api.java.JavaRDD; import org.apache.spark.sql.Dataset; import org.apache.spark.sql.Row; diff --git a/hoodie-utilities/src/main/java/com/uber/hoodie/utilities/exception/HoodieDeltaStreamerException.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/exception/HoodieDeltaStreamerException.java similarity index 91% rename from hoodie-utilities/src/main/java/com/uber/hoodie/utilities/exception/HoodieDeltaStreamerException.java rename to hudi-utilities/src/main/java/org/apache/hudi/utilities/exception/HoodieDeltaStreamerException.java index a570bcd4e..0c7165b72 100644 --- a/hoodie-utilities/src/main/java/com/uber/hoodie/utilities/exception/HoodieDeltaStreamerException.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/exception/HoodieDeltaStreamerException.java @@ -16,9 +16,9 @@ * limitations under the License. */ -package com.uber.hoodie.utilities.exception; +package org.apache.hudi.utilities.exception; -import com.uber.hoodie.exception.HoodieException; +import org.apache.hudi.exception.HoodieException; public class HoodieDeltaStreamerException extends HoodieException { diff --git a/hoodie-utilities/src/main/java/com/uber/hoodie/utilities/exception/HoodieIncrementalPullException.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/exception/HoodieIncrementalPullException.java similarity index 91% rename from hoodie-utilities/src/main/java/com/uber/hoodie/utilities/exception/HoodieIncrementalPullException.java rename to hudi-utilities/src/main/java/org/apache/hudi/utilities/exception/HoodieIncrementalPullException.java index 09f3d12b3..58cad2c2c 100644 --- a/hoodie-utilities/src/main/java/com/uber/hoodie/utilities/exception/HoodieIncrementalPullException.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/exception/HoodieIncrementalPullException.java @@ -16,10 +16,10 @@ * limitations under the License. */ -package com.uber.hoodie.utilities.exception; +package org.apache.hudi.utilities.exception; -import com.uber.hoodie.exception.HoodieException; import java.sql.SQLException; +import org.apache.hudi.exception.HoodieException; public class HoodieIncrementalPullException extends HoodieException { diff --git a/hoodie-utilities/src/main/java/com/uber/hoodie/utilities/exception/HoodieIncrementalPullSQLException.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/exception/HoodieIncrementalPullSQLException.java similarity index 96% rename from hoodie-utilities/src/main/java/com/uber/hoodie/utilities/exception/HoodieIncrementalPullSQLException.java rename to hudi-utilities/src/main/java/org/apache/hudi/utilities/exception/HoodieIncrementalPullSQLException.java index d900e6a8a..230fdea72 100644 --- a/hoodie-utilities/src/main/java/com/uber/hoodie/utilities/exception/HoodieIncrementalPullSQLException.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/exception/HoodieIncrementalPullSQLException.java @@ -16,7 +16,7 @@ * limitations under the License. */ -package com.uber.hoodie.utilities.exception; +package org.apache.hudi.utilities.exception; import java.sql.SQLException; diff --git a/hoodie-utilities/src/main/java/com/uber/hoodie/utilities/keygen/TimestampBasedKeyGenerator.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/keygen/TimestampBasedKeyGenerator.java similarity index 92% rename from hoodie-utilities/src/main/java/com/uber/hoodie/utilities/keygen/TimestampBasedKeyGenerator.java rename to hudi-utilities/src/main/java/org/apache/hudi/utilities/keygen/TimestampBasedKeyGenerator.java index dfd98c1b2..fc41859f4 100644 --- a/hoodie-utilities/src/main/java/com/uber/hoodie/utilities/keygen/TimestampBasedKeyGenerator.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/keygen/TimestampBasedKeyGenerator.java @@ -16,14 +16,8 @@ * limitations under the License. */ -package com.uber.hoodie.utilities.keygen; +package org.apache.hudi.utilities.keygen; -import com.uber.hoodie.DataSourceUtils; -import com.uber.hoodie.SimpleKeyGenerator; -import com.uber.hoodie.common.model.HoodieKey; -import com.uber.hoodie.common.util.TypedProperties; -import com.uber.hoodie.exception.HoodieNotSupportedException; -import com.uber.hoodie.utilities.exception.HoodieDeltaStreamerException; import java.io.Serializable; import java.text.ParseException; import java.text.SimpleDateFormat; @@ -32,6 +26,12 @@ import java.util.Collections; import java.util.Date; import java.util.TimeZone; import org.apache.avro.generic.GenericRecord; +import org.apache.hudi.DataSourceUtils; +import org.apache.hudi.SimpleKeyGenerator; +import org.apache.hudi.common.model.HoodieKey; +import org.apache.hudi.common.util.TypedProperties; +import org.apache.hudi.exception.HoodieNotSupportedException; +import org.apache.hudi.utilities.exception.HoodieDeltaStreamerException; /** * Key generator, that relies on timestamps for partitioning field. Still picks record key by name. diff --git a/hoodie-utilities/src/main/java/com/uber/hoodie/utilities/perf/TimelineServerPerf.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/perf/TimelineServerPerf.java similarity index 95% rename from hoodie-utilities/src/main/java/com/uber/hoodie/utilities/perf/TimelineServerPerf.java rename to hudi-utilities/src/main/java/org/apache/hudi/utilities/perf/TimelineServerPerf.java index e626d27f5..c8469f93e 100644 --- a/hoodie-utilities/src/main/java/com/uber/hoodie/utilities/perf/TimelineServerPerf.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/perf/TimelineServerPerf.java @@ -16,23 +16,13 @@ * limitations under the License. */ -package com.uber.hoodie.utilities.perf; +package org.apache.hudi.utilities.perf; import com.beust.jcommander.JCommander; import com.beust.jcommander.Parameter; import com.codahale.metrics.Histogram; import com.codahale.metrics.Snapshot; import com.codahale.metrics.UniformReservoir; -import com.uber.hoodie.common.model.FileSlice; -import com.uber.hoodie.common.table.HoodieTableMetaClient; -import com.uber.hoodie.common.table.SyncableFileSystemView; -import com.uber.hoodie.common.table.view.FileSystemViewStorageConfig; -import com.uber.hoodie.common.table.view.FileSystemViewStorageType; -import com.uber.hoodie.common.table.view.RemoteHoodieTableFileSystemView; -import com.uber.hoodie.common.util.FSUtils; -import com.uber.hoodie.common.util.Option; -import com.uber.hoodie.timeline.service.TimelineService; -import com.uber.hoodie.utilities.UtilHelpers; import java.io.IOException; import java.io.Serializable; import java.util.ArrayList; @@ -49,6 +39,16 @@ import java.util.stream.IntStream; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hudi.common.model.FileSlice; +import org.apache.hudi.common.table.HoodieTableMetaClient; +import org.apache.hudi.common.table.SyncableFileSystemView; +import org.apache.hudi.common.table.view.FileSystemViewStorageConfig; +import org.apache.hudi.common.table.view.FileSystemViewStorageType; +import org.apache.hudi.common.table.view.RemoteHoodieTableFileSystemView; +import org.apache.hudi.common.util.FSUtils; +import org.apache.hudi.common.util.Option; +import org.apache.hudi.timeline.service.TimelineService; +import org.apache.hudi.utilities.UtilHelpers; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; import org.apache.spark.SparkConf; diff --git a/hoodie-utilities/src/main/java/com/uber/hoodie/utilities/schema/FilebasedSchemaProvider.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/schema/FilebasedSchemaProvider.java similarity index 91% rename from hoodie-utilities/src/main/java/com/uber/hoodie/utilities/schema/FilebasedSchemaProvider.java rename to hudi-utilities/src/main/java/org/apache/hudi/utilities/schema/FilebasedSchemaProvider.java index 63179e4e3..3c9102b44 100644 --- a/hoodie-utilities/src/main/java/com/uber/hoodie/utilities/schema/FilebasedSchemaProvider.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/schema/FilebasedSchemaProvider.java @@ -16,18 +16,17 @@ * limitations under the License. */ -package com.uber.hoodie.utilities.schema; +package org.apache.hudi.utilities.schema; -import com.uber.hoodie.DataSourceUtils; -import com.uber.hoodie.common.util.FSUtils; -import com.uber.hoodie.common.util.TypedProperties; -import com.uber.hoodie.exception.HoodieIOException; import java.io.IOException; import java.util.Collections; - import org.apache.avro.Schema; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hudi.DataSourceUtils; +import org.apache.hudi.common.util.FSUtils; +import org.apache.hudi.common.util.TypedProperties; +import org.apache.hudi.exception.HoodieIOException; import org.apache.spark.api.java.JavaSparkContext; /** diff --git a/hoodie-utilities/src/main/java/com/uber/hoodie/utilities/schema/RowBasedSchemaProvider.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/schema/RowBasedSchemaProvider.java similarity index 94% rename from hoodie-utilities/src/main/java/com/uber/hoodie/utilities/schema/RowBasedSchemaProvider.java rename to hudi-utilities/src/main/java/org/apache/hudi/utilities/schema/RowBasedSchemaProvider.java index fd8df6123..22a36b40a 100644 --- a/hoodie-utilities/src/main/java/com/uber/hoodie/utilities/schema/RowBasedSchemaProvider.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/schema/RowBasedSchemaProvider.java @@ -16,10 +16,10 @@ * limitations under the License. */ -package com.uber.hoodie.utilities.schema; +package org.apache.hudi.utilities.schema; -import com.uber.hoodie.AvroConversionUtils; import org.apache.avro.Schema; +import org.apache.hudi.AvroConversionUtils; import org.apache.spark.sql.types.StructType; public class RowBasedSchemaProvider extends SchemaProvider { diff --git a/hoodie-utilities/src/main/java/com/uber/hoodie/utilities/schema/SchemaProvider.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/schema/SchemaProvider.java similarity index 93% rename from hoodie-utilities/src/main/java/com/uber/hoodie/utilities/schema/SchemaProvider.java rename to hudi-utilities/src/main/java/org/apache/hudi/utilities/schema/SchemaProvider.java index d82b74327..c1f7c349d 100644 --- a/hoodie-utilities/src/main/java/com/uber/hoodie/utilities/schema/SchemaProvider.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/schema/SchemaProvider.java @@ -16,11 +16,11 @@ * limitations under the License. */ -package com.uber.hoodie.utilities.schema; +package org.apache.hudi.utilities.schema; -import com.uber.hoodie.common.util.TypedProperties; import java.io.Serializable; import org.apache.avro.Schema; +import org.apache.hudi.common.util.TypedProperties; import org.apache.spark.api.java.JavaSparkContext; /** diff --git a/hoodie-utilities/src/main/java/com/uber/hoodie/utilities/schema/SchemaRegistryProvider.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/schema/SchemaRegistryProvider.java similarity index 94% rename from hoodie-utilities/src/main/java/com/uber/hoodie/utilities/schema/SchemaRegistryProvider.java rename to hudi-utilities/src/main/java/org/apache/hudi/utilities/schema/SchemaRegistryProvider.java index 68daf7995..7e2ac1641 100644 --- a/hoodie-utilities/src/main/java/com/uber/hoodie/utilities/schema/SchemaRegistryProvider.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/schema/SchemaRegistryProvider.java @@ -16,18 +16,17 @@ * limitations under the License. */ -package com.uber.hoodie.utilities.schema; +package org.apache.hudi.utilities.schema; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; -import com.uber.hoodie.DataSourceUtils; -import com.uber.hoodie.common.util.TypedProperties; -import com.uber.hoodie.exception.HoodieIOException; import java.io.IOException; import java.net.URL; import java.util.Collections; - import org.apache.avro.Schema; +import org.apache.hudi.DataSourceUtils; +import org.apache.hudi.common.util.TypedProperties; +import org.apache.hudi.exception.HoodieIOException; import org.apache.spark.api.java.JavaSparkContext; /** diff --git a/hoodie-utilities/src/main/java/com/uber/hoodie/utilities/sources/AvroDFSSource.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/AvroDFSSource.java similarity index 89% rename from hoodie-utilities/src/main/java/com/uber/hoodie/utilities/sources/AvroDFSSource.java rename to hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/AvroDFSSource.java index 7b802a12c..83e81e80e 100644 --- a/hoodie-utilities/src/main/java/com/uber/hoodie/utilities/sources/AvroDFSSource.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/AvroDFSSource.java @@ -16,17 +16,17 @@ * limitations under the License. */ -package com.uber.hoodie.utilities.sources; +package org.apache.hudi.utilities.sources; -import com.uber.hoodie.common.util.Option; -import com.uber.hoodie.common.util.TypedProperties; -import com.uber.hoodie.common.util.collection.Pair; -import com.uber.hoodie.utilities.schema.SchemaProvider; -import com.uber.hoodie.utilities.sources.helpers.DFSPathSelector; import org.apache.avro.generic.GenericRecord; import org.apache.avro.mapred.AvroKey; import org.apache.avro.mapreduce.AvroKeyInputFormat; import org.apache.hadoop.io.NullWritable; +import org.apache.hudi.common.util.Option; +import org.apache.hudi.common.util.TypedProperties; +import org.apache.hudi.common.util.collection.Pair; +import org.apache.hudi.utilities.schema.SchemaProvider; +import org.apache.hudi.utilities.sources.helpers.DFSPathSelector; import org.apache.spark.api.java.JavaPairRDD; import org.apache.spark.api.java.JavaRDD; import org.apache.spark.api.java.JavaSparkContext; diff --git a/hoodie-utilities/src/main/java/com/uber/hoodie/utilities/sources/AvroKafkaSource.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/AvroKafkaSource.java similarity index 89% rename from hoodie-utilities/src/main/java/com/uber/hoodie/utilities/sources/AvroKafkaSource.java rename to hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/AvroKafkaSource.java index 915030326..3b051ac15 100644 --- a/hoodie-utilities/src/main/java/com/uber/hoodie/utilities/sources/AvroKafkaSource.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/AvroKafkaSource.java @@ -16,16 +16,16 @@ * limitations under the License. */ -package com.uber.hoodie.utilities.sources; +package org.apache.hudi.utilities.sources; -import com.uber.hoodie.common.util.Option; -import com.uber.hoodie.common.util.TypedProperties; -import com.uber.hoodie.utilities.schema.SchemaProvider; -import com.uber.hoodie.utilities.sources.helpers.KafkaOffsetGen; -import com.uber.hoodie.utilities.sources.helpers.KafkaOffsetGen.CheckpointUtils; import io.confluent.kafka.serializers.KafkaAvroDecoder; import kafka.serializer.StringDecoder; import org.apache.avro.generic.GenericRecord; +import org.apache.hudi.common.util.Option; +import org.apache.hudi.common.util.TypedProperties; +import org.apache.hudi.utilities.schema.SchemaProvider; +import org.apache.hudi.utilities.sources.helpers.KafkaOffsetGen; +import org.apache.hudi.utilities.sources.helpers.KafkaOffsetGen.CheckpointUtils; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; import org.apache.spark.api.java.JavaRDD; diff --git a/hoodie-utilities/src/main/java/com/uber/hoodie/utilities/sources/AvroSource.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/AvroSource.java similarity index 89% rename from hoodie-utilities/src/main/java/com/uber/hoodie/utilities/sources/AvroSource.java rename to hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/AvroSource.java index 351b25e90..815864029 100644 --- a/hoodie-utilities/src/main/java/com/uber/hoodie/utilities/sources/AvroSource.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/AvroSource.java @@ -16,11 +16,11 @@ * limitations under the License. */ -package com.uber.hoodie.utilities.sources; +package org.apache.hudi.utilities.sources; -import com.uber.hoodie.common.util.TypedProperties; -import com.uber.hoodie.utilities.schema.SchemaProvider; import org.apache.avro.generic.GenericRecord; +import org.apache.hudi.common.util.TypedProperties; +import org.apache.hudi.utilities.schema.SchemaProvider; import org.apache.spark.api.java.JavaRDD; import org.apache.spark.api.java.JavaSparkContext; import org.apache.spark.sql.SparkSession; diff --git a/hoodie-utilities/src/main/java/com/uber/hoodie/utilities/sources/HiveIncrPullSource.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/HiveIncrPullSource.java similarity index 91% rename from hoodie-utilities/src/main/java/com/uber/hoodie/utilities/sources/HiveIncrPullSource.java rename to hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/HiveIncrPullSource.java index ca681f5d4..97a5bac79 100644 --- a/hoodie-utilities/src/main/java/com/uber/hoodie/utilities/sources/HiveIncrPullSource.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/HiveIncrPullSource.java @@ -16,14 +16,8 @@ * limitations under the License. */ -package com.uber.hoodie.utilities.sources; +package org.apache.hudi.utilities.sources; -import com.uber.hoodie.DataSourceUtils; -import com.uber.hoodie.common.util.FSUtils; -import com.uber.hoodie.common.util.Option; -import com.uber.hoodie.common.util.TypedProperties; -import com.uber.hoodie.exception.HoodieIOException; -import com.uber.hoodie.utilities.schema.SchemaProvider; import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; @@ -37,6 +31,13 @@ import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.NullWritable; +import org.apache.hudi.DataSourceUtils; +import org.apache.hudi.common.util.FSUtils; +import org.apache.hudi.common.util.Option; +import org.apache.hudi.common.util.TypedProperties; +import org.apache.hudi.exception.HoodieIOException; +import org.apache.hudi.utilities.HiveIncrementalPuller; +import org.apache.hudi.utilities.schema.SchemaProvider; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; import org.apache.spark.api.java.JavaPairRDD; @@ -45,7 +46,7 @@ import org.apache.spark.api.java.JavaSparkContext; import org.apache.spark.sql.SparkSession; /** - * Source to read deltas produced by {@link com.uber.hoodie.utilities.HiveIncrementalPuller}, commit by commit and apply + * Source to read deltas produced by {@link HiveIncrementalPuller}, commit by commit and apply * to the target table *

* The general idea here is to have commits sync across the data pipeline. diff --git a/hoodie-utilities/src/main/java/com/uber/hoodie/utilities/sources/HoodieIncrSource.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/HoodieIncrSource.java similarity index 92% rename from hoodie-utilities/src/main/java/com/uber/hoodie/utilities/sources/HoodieIncrSource.java rename to hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/HoodieIncrSource.java index b0fa59b72..5d95e7d4a 100644 --- a/hoodie-utilities/src/main/java/com/uber/hoodie/utilities/sources/HoodieIncrSource.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/HoodieIncrSource.java @@ -16,18 +16,18 @@ * limitations under the License. */ -package com.uber.hoodie.utilities.sources; +package org.apache.hudi.utilities.sources; -import com.uber.hoodie.DataSourceReadOptions; -import com.uber.hoodie.DataSourceUtils; -import com.uber.hoodie.common.model.HoodieRecord; -import com.uber.hoodie.common.util.Option; -import com.uber.hoodie.common.util.TypedProperties; -import com.uber.hoodie.common.util.collection.Pair; -import com.uber.hoodie.hive.SlashEncodedDayPartitionValueExtractor; -import com.uber.hoodie.utilities.schema.SchemaProvider; -import com.uber.hoodie.utilities.sources.helpers.IncrSourceHelper; import java.util.Arrays; +import org.apache.hudi.DataSourceReadOptions; +import org.apache.hudi.DataSourceUtils; +import org.apache.hudi.common.model.HoodieRecord; +import org.apache.hudi.common.util.Option; +import org.apache.hudi.common.util.TypedProperties; +import org.apache.hudi.common.util.collection.Pair; +import org.apache.hudi.hive.SlashEncodedDayPartitionValueExtractor; +import org.apache.hudi.utilities.schema.SchemaProvider; +import org.apache.hudi.utilities.sources.helpers.IncrSourceHelper; import org.apache.spark.api.java.JavaSparkContext; import org.apache.spark.sql.DataFrameReader; import org.apache.spark.sql.Dataset; @@ -109,7 +109,7 @@ public class HoodieIncrSource extends RowSource { } // Do Incr pull. Set end instant if available - DataFrameReader reader = sparkSession.read().format("com.uber.hoodie") + DataFrameReader reader = sparkSession.read().format("org.apache.hudi") .option(DataSourceReadOptions.VIEW_TYPE_OPT_KEY(), DataSourceReadOptions.VIEW_TYPE_INCREMENTAL_OPT_VAL()) .option(DataSourceReadOptions.BEGIN_INSTANTTIME_OPT_KEY(), instantEndpts.getLeft()) .option(DataSourceReadOptions.END_INSTANTTIME_OPT_KEY(), instantEndpts.getRight()); diff --git a/hoodie-utilities/src/main/java/com/uber/hoodie/utilities/sources/InputBatch.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/InputBatch.java similarity index 91% rename from hoodie-utilities/src/main/java/com/uber/hoodie/utilities/sources/InputBatch.java rename to hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/InputBatch.java index 8625cfac3..efb8dbeda 100644 --- a/hoodie-utilities/src/main/java/com/uber/hoodie/utilities/sources/InputBatch.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/InputBatch.java @@ -16,10 +16,10 @@ * limitations under the License. */ -package com.uber.hoodie.utilities.sources; +package org.apache.hudi.utilities.sources; -import com.uber.hoodie.common.util.Option; -import com.uber.hoodie.utilities.schema.SchemaProvider; +import org.apache.hudi.common.util.Option; +import org.apache.hudi.utilities.schema.SchemaProvider; public class InputBatch { diff --git a/hoodie-utilities/src/main/java/com/uber/hoodie/utilities/sources/JsonDFSSource.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/JsonDFSSource.java similarity index 86% rename from hoodie-utilities/src/main/java/com/uber/hoodie/utilities/sources/JsonDFSSource.java rename to hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/JsonDFSSource.java index 5f17acb75..a801d2a4b 100644 --- a/hoodie-utilities/src/main/java/com/uber/hoodie/utilities/sources/JsonDFSSource.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/JsonDFSSource.java @@ -16,13 +16,13 @@ * limitations under the License. */ -package com.uber.hoodie.utilities.sources; +package org.apache.hudi.utilities.sources; -import com.uber.hoodie.common.util.Option; -import com.uber.hoodie.common.util.TypedProperties; -import com.uber.hoodie.common.util.collection.Pair; -import com.uber.hoodie.utilities.schema.SchemaProvider; -import com.uber.hoodie.utilities.sources.helpers.DFSPathSelector; +import org.apache.hudi.common.util.Option; +import org.apache.hudi.common.util.TypedProperties; +import org.apache.hudi.common.util.collection.Pair; +import org.apache.hudi.utilities.schema.SchemaProvider; +import org.apache.hudi.utilities.sources.helpers.DFSPathSelector; import org.apache.spark.api.java.JavaRDD; import org.apache.spark.api.java.JavaSparkContext; import org.apache.spark.sql.SparkSession; diff --git a/hoodie-utilities/src/main/java/com/uber/hoodie/utilities/sources/JsonKafkaSource.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/JsonKafkaSource.java similarity index 88% rename from hoodie-utilities/src/main/java/com/uber/hoodie/utilities/sources/JsonKafkaSource.java rename to hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/JsonKafkaSource.java index 665a1dccc..867798dcb 100644 --- a/hoodie-utilities/src/main/java/com/uber/hoodie/utilities/sources/JsonKafkaSource.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/JsonKafkaSource.java @@ -16,14 +16,14 @@ * limitations under the License. */ -package com.uber.hoodie.utilities.sources; +package org.apache.hudi.utilities.sources; -import com.uber.hoodie.common.util.Option; -import com.uber.hoodie.common.util.TypedProperties; -import com.uber.hoodie.utilities.schema.SchemaProvider; -import com.uber.hoodie.utilities.sources.helpers.KafkaOffsetGen; -import com.uber.hoodie.utilities.sources.helpers.KafkaOffsetGen.CheckpointUtils; import kafka.serializer.StringDecoder; +import org.apache.hudi.common.util.Option; +import org.apache.hudi.common.util.TypedProperties; +import org.apache.hudi.utilities.schema.SchemaProvider; +import org.apache.hudi.utilities.sources.helpers.KafkaOffsetGen; +import org.apache.hudi.utilities.sources.helpers.KafkaOffsetGen.CheckpointUtils; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; import org.apache.spark.api.java.JavaRDD; diff --git a/hoodie-utilities/src/main/java/com/uber/hoodie/utilities/sources/JsonSource.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/JsonSource.java similarity index 89% rename from hoodie-utilities/src/main/java/com/uber/hoodie/utilities/sources/JsonSource.java rename to hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/JsonSource.java index 816175538..ac47c956b 100644 --- a/hoodie-utilities/src/main/java/com/uber/hoodie/utilities/sources/JsonSource.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/JsonSource.java @@ -16,10 +16,10 @@ * limitations under the License. */ -package com.uber.hoodie.utilities.sources; +package org.apache.hudi.utilities.sources; -import com.uber.hoodie.common.util.TypedProperties; -import com.uber.hoodie.utilities.schema.SchemaProvider; +import org.apache.hudi.common.util.TypedProperties; +import org.apache.hudi.utilities.schema.SchemaProvider; import org.apache.spark.api.java.JavaRDD; import org.apache.spark.api.java.JavaSparkContext; import org.apache.spark.sql.SparkSession; diff --git a/hoodie-utilities/src/main/java/com/uber/hoodie/utilities/sources/RowSource.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/RowSource.java similarity index 85% rename from hoodie-utilities/src/main/java/com/uber/hoodie/utilities/sources/RowSource.java rename to hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/RowSource.java index 286943135..715ba8028 100644 --- a/hoodie-utilities/src/main/java/com/uber/hoodie/utilities/sources/RowSource.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/RowSource.java @@ -16,13 +16,13 @@ * limitations under the License. */ -package com.uber.hoodie.utilities.sources; +package org.apache.hudi.utilities.sources; -import com.uber.hoodie.common.util.Option; -import com.uber.hoodie.common.util.TypedProperties; -import com.uber.hoodie.common.util.collection.Pair; -import com.uber.hoodie.utilities.schema.RowBasedSchemaProvider; -import com.uber.hoodie.utilities.schema.SchemaProvider; +import org.apache.hudi.common.util.Option; +import org.apache.hudi.common.util.TypedProperties; +import org.apache.hudi.common.util.collection.Pair; +import org.apache.hudi.utilities.schema.RowBasedSchemaProvider; +import org.apache.hudi.utilities.schema.SchemaProvider; import org.apache.spark.api.java.JavaSparkContext; import org.apache.spark.sql.Dataset; import org.apache.spark.sql.Row; diff --git a/hoodie-utilities/src/main/java/com/uber/hoodie/utilities/sources/Source.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/Source.java similarity index 93% rename from hoodie-utilities/src/main/java/com/uber/hoodie/utilities/sources/Source.java rename to hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/Source.java index 1c737d72a..2900ae09d 100644 --- a/hoodie-utilities/src/main/java/com/uber/hoodie/utilities/sources/Source.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/Source.java @@ -16,12 +16,12 @@ * limitations under the License. */ -package com.uber.hoodie.utilities.sources; +package org.apache.hudi.utilities.sources; -import com.uber.hoodie.common.util.Option; -import com.uber.hoodie.common.util.TypedProperties; -import com.uber.hoodie.utilities.schema.SchemaProvider; import java.io.Serializable; +import org.apache.hudi.common.util.Option; +import org.apache.hudi.common.util.TypedProperties; +import org.apache.hudi.utilities.schema.SchemaProvider; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; import org.apache.spark.api.java.JavaSparkContext; diff --git a/hoodie-utilities/src/main/java/com/uber/hoodie/utilities/sources/helpers/AvroConvertor.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/helpers/AvroConvertor.java similarity index 96% rename from hoodie-utilities/src/main/java/com/uber/hoodie/utilities/sources/helpers/AvroConvertor.java rename to hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/helpers/AvroConvertor.java index 90c341db8..eeaf3f6f2 100644 --- a/hoodie-utilities/src/main/java/com/uber/hoodie/utilities/sources/helpers/AvroConvertor.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/helpers/AvroConvertor.java @@ -16,15 +16,15 @@ * limitations under the License. */ -package com.uber.hoodie.utilities.sources.helpers; +package org.apache.hudi.utilities.sources.helpers; import com.twitter.bijection.Injection; import com.twitter.bijection.avro.GenericAvroCodecs; -import com.uber.hoodie.avro.MercifulJsonConverter; import java.io.IOException; import java.io.Serializable; import org.apache.avro.Schema; import org.apache.avro.generic.GenericRecord; +import org.apache.hudi.avro.MercifulJsonConverter; /** * Convert a variety of datum into Avro GenericRecords. Has a bunch of lazy diff --git a/hoodie-utilities/src/main/java/com/uber/hoodie/utilities/sources/helpers/DFSPathSelector.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/helpers/DFSPathSelector.java similarity index 88% rename from hoodie-utilities/src/main/java/com/uber/hoodie/utilities/sources/helpers/DFSPathSelector.java rename to hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/helpers/DFSPathSelector.java index 26d2e2da1..fd5c67682 100644 --- a/hoodie-utilities/src/main/java/com/uber/hoodie/utilities/sources/helpers/DFSPathSelector.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/helpers/DFSPathSelector.java @@ -16,17 +16,13 @@ * limitations under the License. */ -package com.uber.hoodie.utilities.sources.helpers; +package org.apache.hudi.utilities.sources.helpers; -import com.uber.hoodie.DataSourceUtils; -import com.uber.hoodie.common.util.FSUtils; -import com.uber.hoodie.common.util.Option; -import com.uber.hoodie.common.util.TypedProperties; -import com.uber.hoodie.common.util.collection.ImmutablePair; -import com.uber.hoodie.common.util.collection.Pair; -import com.uber.hoodie.exception.HoodieIOException; import java.io.IOException; -import java.util.*; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Comparator; +import java.util.List; import java.util.stream.Collectors; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; @@ -34,6 +30,13 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.LocatedFileStatus; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.RemoteIterator; +import org.apache.hudi.DataSourceUtils; +import org.apache.hudi.common.util.FSUtils; +import org.apache.hudi.common.util.Option; +import org.apache.hudi.common.util.TypedProperties; +import org.apache.hudi.common.util.collection.ImmutablePair; +import org.apache.hudi.common.util.collection.Pair; +import org.apache.hudi.exception.HoodieIOException; public class DFSPathSelector { diff --git a/hoodie-utilities/src/main/java/com/uber/hoodie/utilities/sources/helpers/IncrSourceHelper.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/helpers/IncrSourceHelper.java similarity index 93% rename from hoodie-utilities/src/main/java/com/uber/hoodie/utilities/sources/helpers/IncrSourceHelper.java rename to hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/helpers/IncrSourceHelper.java index 359c772c2..021c8ee97 100644 --- a/hoodie-utilities/src/main/java/com/uber/hoodie/utilities/sources/helpers/IncrSourceHelper.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/helpers/IncrSourceHelper.java @@ -16,14 +16,14 @@ * limitations under the License. */ -package com.uber.hoodie.utilities.sources.helpers; +package org.apache.hudi.utilities.sources.helpers; import com.google.common.base.Preconditions; -import com.uber.hoodie.common.table.HoodieTableMetaClient; -import com.uber.hoodie.common.table.HoodieTimeline; -import com.uber.hoodie.common.table.timeline.HoodieInstant; -import com.uber.hoodie.common.util.Option; -import com.uber.hoodie.common.util.collection.Pair; +import org.apache.hudi.common.table.HoodieTableMetaClient; +import org.apache.hudi.common.table.HoodieTimeline; +import org.apache.hudi.common.table.timeline.HoodieInstant; +import org.apache.hudi.common.util.Option; +import org.apache.hudi.common.util.collection.Pair; import org.apache.spark.api.java.JavaSparkContext; import org.apache.spark.sql.Row; diff --git a/hoodie-utilities/src/main/java/com/uber/hoodie/utilities/sources/helpers/KafkaOffsetGen.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/helpers/KafkaOffsetGen.java similarity index 97% rename from hoodie-utilities/src/main/java/com/uber/hoodie/utilities/sources/helpers/KafkaOffsetGen.java rename to hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/helpers/KafkaOffsetGen.java index ae1ef2037..2b0fd8791 100644 --- a/hoodie-utilities/src/main/java/com/uber/hoodie/utilities/sources/helpers/KafkaOffsetGen.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/helpers/KafkaOffsetGen.java @@ -16,13 +16,8 @@ * limitations under the License. */ -package com.uber.hoodie.utilities.sources.helpers; +package org.apache.hudi.utilities.sources.helpers; -import com.uber.hoodie.DataSourceUtils; -import com.uber.hoodie.common.util.Option; -import com.uber.hoodie.common.util.TypedProperties; -import com.uber.hoodie.exception.HoodieNotSupportedException; -import com.uber.hoodie.utilities.exception.HoodieDeltaStreamerException; import java.util.Arrays; import java.util.Collections; import java.util.Comparator; @@ -30,6 +25,11 @@ import java.util.HashMap; import java.util.HashSet; import java.util.stream.Collectors; import kafka.common.TopicAndPartition; +import org.apache.hudi.DataSourceUtils; +import org.apache.hudi.common.util.Option; +import org.apache.hudi.common.util.TypedProperties; +import org.apache.hudi.exception.HoodieNotSupportedException; +import org.apache.hudi.utilities.exception.HoodieDeltaStreamerException; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; import org.apache.spark.streaming.kafka.KafkaCluster; diff --git a/hoodie-utilities/src/main/java/com/uber/hoodie/utilities/transform/IdentityTransformer.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/transform/IdentityTransformer.java similarity index 92% rename from hoodie-utilities/src/main/java/com/uber/hoodie/utilities/transform/IdentityTransformer.java rename to hudi-utilities/src/main/java/org/apache/hudi/utilities/transform/IdentityTransformer.java index dc1b78e5d..5a2c2c999 100644 --- a/hoodie-utilities/src/main/java/com/uber/hoodie/utilities/transform/IdentityTransformer.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/transform/IdentityTransformer.java @@ -16,9 +16,9 @@ * limitations under the License. */ -package com.uber.hoodie.utilities.transform; +package org.apache.hudi.utilities.transform; -import com.uber.hoodie.common.util.TypedProperties; +import org.apache.hudi.common.util.TypedProperties; import org.apache.spark.api.java.JavaSparkContext; import org.apache.spark.sql.Dataset; import org.apache.spark.sql.Row; diff --git a/hoodie-utilities/src/main/java/com/uber/hoodie/utilities/transform/SqlQueryBasedTransformer.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/transform/SqlQueryBasedTransformer.java similarity index 96% rename from hoodie-utilities/src/main/java/com/uber/hoodie/utilities/transform/SqlQueryBasedTransformer.java rename to hudi-utilities/src/main/java/org/apache/hudi/utilities/transform/SqlQueryBasedTransformer.java index bb38b7037..ce78deb1d 100644 --- a/hoodie-utilities/src/main/java/com/uber/hoodie/utilities/transform/SqlQueryBasedTransformer.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/transform/SqlQueryBasedTransformer.java @@ -16,10 +16,10 @@ * limitations under the License. */ -package com.uber.hoodie.utilities.transform; +package org.apache.hudi.utilities.transform; -import com.uber.hoodie.common.util.TypedProperties; import java.util.UUID; +import org.apache.hudi.common.util.TypedProperties; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; import org.apache.spark.api.java.JavaSparkContext; diff --git a/hoodie-utilities/src/main/java/com/uber/hoodie/utilities/transform/Transformer.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/transform/Transformer.java similarity index 93% rename from hoodie-utilities/src/main/java/com/uber/hoodie/utilities/transform/Transformer.java rename to hudi-utilities/src/main/java/org/apache/hudi/utilities/transform/Transformer.java index 5f2fc876a..6983ec3d7 100644 --- a/hoodie-utilities/src/main/java/com/uber/hoodie/utilities/transform/Transformer.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/transform/Transformer.java @@ -16,9 +16,9 @@ * limitations under the License. */ -package com.uber.hoodie.utilities.transform; +package org.apache.hudi.utilities.transform; -import com.uber.hoodie.common.util.TypedProperties; +import org.apache.hudi.common.util.TypedProperties; import org.apache.spark.api.java.JavaSparkContext; import org.apache.spark.sql.Dataset; import org.apache.spark.sql.Row; diff --git a/hoodie-utilities/src/main/resources/META-INF/LICENSE.txt b/hudi-utilities/src/main/resources/META-INF/LICENSE.txt similarity index 100% rename from hoodie-utilities/src/main/resources/META-INF/LICENSE.txt rename to hudi-utilities/src/main/resources/META-INF/LICENSE.txt diff --git a/hoodie-utilities/src/main/resources/META-INF/NOTICE.txt b/hudi-utilities/src/main/resources/META-INF/NOTICE.txt similarity index 100% rename from hoodie-utilities/src/main/resources/META-INF/NOTICE.txt rename to hudi-utilities/src/main/resources/META-INF/NOTICE.txt diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/SchedulerConfGeneratorTest.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/SchedulerConfGeneratorTest.java new file mode 100644 index 000000000..091c43651 --- /dev/null +++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/SchedulerConfGeneratorTest.java @@ -0,0 +1,35 @@ +package org.apache.hudi.utilities; + +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; + +import java.util.Map; +import org.apache.hudi.common.model.HoodieTableType; +import org.apache.hudi.utilities.deltastreamer.HoodieDeltaStreamer; +import org.apache.hudi.utilities.deltastreamer.SchedulerConfGenerator; +import org.junit.Test; + +public class SchedulerConfGeneratorTest { + + @Test + public void testGenerateSparkSchedulingConf() throws Exception { + HoodieDeltaStreamer.Config cfg = new HoodieDeltaStreamer.Config(); + Map configs = SchedulerConfGenerator.getSparkSchedulingConfigs(cfg); + assertNull("spark.scheduler.mode not set", configs.get(SchedulerConfGenerator.SPARK_SCHEDULER_ALLOCATION_FILE_KEY)); + + System.setProperty(SchedulerConfGenerator.SPARK_SCHEDULER_MODE_KEY, "FAIR"); + cfg.continuousMode = false; + configs = SchedulerConfGenerator.getSparkSchedulingConfigs(cfg); + assertNull("continuousMode is false", configs.get(SchedulerConfGenerator.SPARK_SCHEDULER_ALLOCATION_FILE_KEY)); + + cfg.continuousMode = true; + cfg.storageType = HoodieTableType.COPY_ON_WRITE.name(); + configs = SchedulerConfGenerator.getSparkSchedulingConfigs(cfg); + assertNull("storageType is not MERGE_ON_READ", configs.get( + SchedulerConfGenerator.SPARK_SCHEDULER_ALLOCATION_FILE_KEY)); + + cfg.storageType = HoodieTableType.MERGE_ON_READ.name(); + configs = SchedulerConfGenerator.getSparkSchedulingConfigs(cfg); + assertNotNull("all satisfies", configs.get(SchedulerConfGenerator.SPARK_SCHEDULER_ALLOCATION_FILE_KEY)); + } +} diff --git a/hoodie-utilities/src/test/java/com/uber/hoodie/utilities/TestHDFSParquetImporter.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/TestHDFSParquetImporter.java similarity index 96% rename from hoodie-utilities/src/test/java/com/uber/hoodie/utilities/TestHDFSParquetImporter.java rename to hudi-utilities/src/test/java/org/apache/hudi/utilities/TestHDFSParquetImporter.java index 2130a11f2..6907dbcc7 100644 --- a/hoodie-utilities/src/test/java/com/uber/hoodie/utilities/TestHDFSParquetImporter.java +++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/TestHDFSParquetImporter.java @@ -16,18 +16,11 @@ * limitations under the License. */ -package com.uber.hoodie.utilities; +package org.apache.hudi.utilities; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; -import com.uber.hoodie.HoodieReadClient; -import com.uber.hoodie.HoodieWriteClient; -import com.uber.hoodie.common.HoodieTestDataGenerator; -import com.uber.hoodie.common.minicluster.HdfsTestService; -import com.uber.hoodie.common.model.HoodieTestUtils; -import com.uber.hoodie.common.table.HoodieTimeline; -import com.uber.hoodie.common.table.timeline.HoodieActiveTimeline; import java.io.IOException; import java.io.Serializable; import java.text.ParseException; @@ -45,6 +38,13 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.RemoteIterator; import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.hudi.HoodieReadClient; +import org.apache.hudi.HoodieWriteClient; +import org.apache.hudi.common.HoodieTestDataGenerator; +import org.apache.hudi.common.minicluster.HdfsTestService; +import org.apache.hudi.common.model.HoodieTestUtils; +import org.apache.hudi.common.table.HoodieTimeline; +import org.apache.hudi.common.table.timeline.HoodieActiveTimeline; import org.apache.parquet.avro.AvroParquetWriter; import org.apache.parquet.hadoop.ParquetWriter; import org.apache.spark.SparkConf; diff --git a/hoodie-utilities/src/test/java/com/uber/hoodie/utilities/TestHoodieDeltaStreamer.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/TestHoodieDeltaStreamer.java similarity index 93% rename from hoodie-utilities/src/test/java/com/uber/hoodie/utilities/TestHoodieDeltaStreamer.java rename to hudi-utilities/src/test/java/org/apache/hudi/utilities/TestHoodieDeltaStreamer.java index 7fe187e01..cce1034bd 100644 --- a/hoodie-utilities/src/test/java/com/uber/hoodie/utilities/TestHoodieDeltaStreamer.java +++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/TestHoodieDeltaStreamer.java @@ -16,37 +16,12 @@ * limitations under the License. */ -package com.uber.hoodie.utilities; +package org.apache.hudi.utilities; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; -import com.uber.hoodie.DataSourceWriteOptions; -import com.uber.hoodie.SimpleKeyGenerator; -import com.uber.hoodie.common.model.HoodieCommitMetadata; -import com.uber.hoodie.common.model.HoodieTableType; -import com.uber.hoodie.common.table.HoodieTableMetaClient; -import com.uber.hoodie.common.table.HoodieTimeline; -import com.uber.hoodie.common.table.timeline.HoodieInstant; -import com.uber.hoodie.common.util.DFSPropertiesConfiguration; -import com.uber.hoodie.common.util.Option; -import com.uber.hoodie.common.util.TypedProperties; -import com.uber.hoodie.config.HoodieCompactionConfig; -import com.uber.hoodie.exception.DatasetNotFoundException; -import com.uber.hoodie.hive.HiveSyncConfig; -import com.uber.hoodie.hive.HoodieHiveClient; -import com.uber.hoodie.hive.MultiPartKeysValueExtractor; -import com.uber.hoodie.utilities.deltastreamer.HoodieDeltaStreamer; -import com.uber.hoodie.utilities.deltastreamer.HoodieDeltaStreamer.Operation; -import com.uber.hoodie.utilities.schema.FilebasedSchemaProvider; -import com.uber.hoodie.utilities.sources.DistributedTestDataSource; -import com.uber.hoodie.utilities.sources.HoodieIncrSource; -import com.uber.hoodie.utilities.sources.InputBatch; -import com.uber.hoodie.utilities.sources.TestDataSource; -import com.uber.hoodie.utilities.sources.config.TestSourceConfig; -import com.uber.hoodie.utilities.transform.SqlQueryBasedTransformer; -import com.uber.hoodie.utilities.transform.Transformer; import java.io.IOException; import java.util.ArrayList; import java.util.List; @@ -58,6 +33,31 @@ import java.util.stream.Collectors; import org.apache.avro.generic.GenericRecord; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hudi.DataSourceWriteOptions; +import org.apache.hudi.SimpleKeyGenerator; +import org.apache.hudi.common.model.HoodieCommitMetadata; +import org.apache.hudi.common.model.HoodieTableType; +import org.apache.hudi.common.table.HoodieTableMetaClient; +import org.apache.hudi.common.table.HoodieTimeline; +import org.apache.hudi.common.table.timeline.HoodieInstant; +import org.apache.hudi.common.util.DFSPropertiesConfiguration; +import org.apache.hudi.common.util.Option; +import org.apache.hudi.common.util.TypedProperties; +import org.apache.hudi.config.HoodieCompactionConfig; +import org.apache.hudi.exception.DatasetNotFoundException; +import org.apache.hudi.hive.HiveSyncConfig; +import org.apache.hudi.hive.HoodieHiveClient; +import org.apache.hudi.hive.MultiPartKeysValueExtractor; +import org.apache.hudi.utilities.deltastreamer.HoodieDeltaStreamer; +import org.apache.hudi.utilities.deltastreamer.HoodieDeltaStreamer.Operation; +import org.apache.hudi.utilities.schema.FilebasedSchemaProvider; +import org.apache.hudi.utilities.sources.DistributedTestDataSource; +import org.apache.hudi.utilities.sources.HoodieIncrSource; +import org.apache.hudi.utilities.sources.InputBatch; +import org.apache.hudi.utilities.sources.TestDataSource; +import org.apache.hudi.utilities.sources.config.TestSourceConfig; +import org.apache.hudi.utilities.transform.SqlQueryBasedTransformer; +import org.apache.hudi.utilities.transform.Transformer; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; import org.apache.spark.api.java.JavaRDD; @@ -77,7 +77,7 @@ import org.junit.BeforeClass; import org.junit.Test; /** - * Basic tests against {@link com.uber.hoodie.utilities.deltastreamer.HoodieDeltaStreamer}, by issuing bulk_inserts, + * Basic tests against {@link HoodieDeltaStreamer}, by issuing bulk_inserts, * upserts, inserts. Check counts at the end. */ public class TestHoodieDeltaStreamer extends UtilitiesTestBase { @@ -202,24 +202,24 @@ public class TestHoodieDeltaStreamer extends UtilitiesTestBase { } static void assertRecordCount(long expected, String datasetPath, SQLContext sqlContext) { - long recordCount = sqlContext.read().format("com.uber.hoodie").load(datasetPath).count(); + long recordCount = sqlContext.read().format("org.apache.hudi").load(datasetPath).count(); assertEquals(expected, recordCount); } static List countsPerCommit(String datasetPath, SQLContext sqlContext) { - return sqlContext.read().format("com.uber.hoodie").load(datasetPath).groupBy("_hoodie_commit_time").count() + return sqlContext.read().format("org.apache.hudi").load(datasetPath).groupBy("_hoodie_commit_time").count() .sort("_hoodie_commit_time").collectAsList(); } static void assertDistanceCount(long expected, String datasetPath, SQLContext sqlContext) { - sqlContext.read().format("com.uber.hoodie").load(datasetPath).registerTempTable("tmp_trips"); + sqlContext.read().format("org.apache.hudi").load(datasetPath).registerTempTable("tmp_trips"); long recordCount = sqlContext.sparkSession().sql("select * from tmp_trips where haversine_distance is not NULL").count(); assertEquals(expected, recordCount); } static void assertDistanceCountWithExactValue(long expected, String datasetPath, SQLContext sqlContext) { - sqlContext.read().format("com.uber.hoodie").load(datasetPath).registerTempTable("tmp_trips"); + sqlContext.read().format("org.apache.hudi").load(datasetPath).registerTempTable("tmp_trips"); long recordCount = sqlContext.sparkSession().sql("select * from tmp_trips where haversine_distance = 1.0").count(); assertEquals(expected, recordCount); @@ -278,7 +278,7 @@ public class TestHoodieDeltaStreamer extends UtilitiesTestBase { assertEquals(2, props.getInteger("hoodie.upsert.shuffle.parallelism")); assertEquals("_row_key", props.getString("hoodie.datasource.write.recordkey.field")); assertEquals( - "com.uber.hoodie.utilities.TestHoodieDeltaStreamer$TestGenerator", + "org.apache.hudi.utilities.TestHoodieDeltaStreamer$TestGenerator", props.getString("hoodie.datasource.write.keygenerator.class") ); } diff --git a/hoodie-utilities/src/test/java/com/uber/hoodie/utilities/TestHoodieSnapshotCopier.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/TestHoodieSnapshotCopier.java similarity index 97% rename from hoodie-utilities/src/test/java/com/uber/hoodie/utilities/TestHoodieSnapshotCopier.java rename to hudi-utilities/src/test/java/org/apache/hudi/utilities/TestHoodieSnapshotCopier.java index 6de320ba7..00102caea 100644 --- a/hoodie-utilities/src/test/java/com/uber/hoodie/utilities/TestHoodieSnapshotCopier.java +++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/TestHoodieSnapshotCopier.java @@ -16,20 +16,20 @@ * limitations under the License. */ -package com.uber.hoodie.utilities; +package org.apache.hudi.utilities; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; -import com.uber.hoodie.common.HoodieTestDataGenerator; -import com.uber.hoodie.common.model.HoodieTestUtils; -import com.uber.hoodie.common.util.FSUtils; import java.io.File; import java.io.IOException; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hudi.common.HoodieTestDataGenerator; +import org.apache.hudi.common.model.HoodieTestUtils; +import org.apache.hudi.common.util.FSUtils; import org.apache.spark.SparkConf; import org.apache.spark.api.java.JavaSparkContext; import org.junit.After; diff --git a/hoodie-utilities/src/test/java/com/uber/hoodie/utilities/UtilitiesTestBase.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/UtilitiesTestBase.java similarity index 91% rename from hoodie-utilities/src/test/java/com/uber/hoodie/utilities/UtilitiesTestBase.java rename to hudi-utilities/src/test/java/org/apache/hudi/utilities/UtilitiesTestBase.java index 971a9b718..2aec7dc16 100644 --- a/hoodie-utilities/src/test/java/com/uber/hoodie/utilities/UtilitiesTestBase.java +++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/UtilitiesTestBase.java @@ -16,19 +16,9 @@ * limitations under the License. */ -package com.uber.hoodie.utilities; +package org.apache.hudi.utilities; import com.google.common.collect.ImmutableList; -import com.uber.hoodie.common.TestRawTripPayload; -import com.uber.hoodie.common.minicluster.HdfsTestService; -import com.uber.hoodie.common.model.HoodieRecord; -import com.uber.hoodie.common.model.HoodieTableType; -import com.uber.hoodie.common.table.HoodieTableMetaClient; -import com.uber.hoodie.common.util.TypedProperties; -import com.uber.hoodie.hive.HiveSyncConfig; -import com.uber.hoodie.hive.HoodieHiveClient; -import com.uber.hoodie.hive.util.HiveTestService; -import com.uber.hoodie.utilities.sources.TestDataSource; import java.io.BufferedReader; import java.io.IOException; import java.io.InputStreamReader; @@ -40,6 +30,16 @@ import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hive.service.server.HiveServer2; +import org.apache.hudi.common.TestRawTripPayload; +import org.apache.hudi.common.minicluster.HdfsTestService; +import org.apache.hudi.common.model.HoodieRecord; +import org.apache.hudi.common.model.HoodieTableType; +import org.apache.hudi.common.table.HoodieTableMetaClient; +import org.apache.hudi.common.util.TypedProperties; +import org.apache.hudi.hive.HiveSyncConfig; +import org.apache.hudi.hive.HoodieHiveClient; +import org.apache.hudi.hive.util.HiveTestService; +import org.apache.hudi.utilities.sources.TestDataSource; import org.apache.spark.api.java.JavaSparkContext; import org.apache.spark.sql.SQLContext; import org.apache.spark.sql.SparkSession; @@ -123,6 +123,7 @@ public class UtilitiesTestBase { hiveSyncConfig.tableName = tableName; hiveSyncConfig.basePath = basePath; hiveSyncConfig.assumeDatePartitioning = false; + hiveSyncConfig.usePreApacheInputFormat = false; hiveSyncConfig.partitionFields = new ImmutableList.Builder().add("datestr").build(); return hiveSyncConfig; } diff --git a/hoodie-utilities/src/test/java/com/uber/hoodie/utilities/sources/AbstractBaseTestSource.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/AbstractBaseTestSource.java similarity index 90% rename from hoodie-utilities/src/test/java/com/uber/hoodie/utilities/sources/AbstractBaseTestSource.java rename to hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/AbstractBaseTestSource.java index 3016c6730..b9725d4b8 100644 --- a/hoodie-utilities/src/test/java/com/uber/hoodie/utilities/sources/AbstractBaseTestSource.java +++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/AbstractBaseTestSource.java @@ -1,13 +1,5 @@ -package com.uber.hoodie.utilities.sources; +package org.apache.hudi.utilities.sources; -import com.uber.hoodie.common.HoodieTestDataGenerator; -import com.uber.hoodie.common.model.HoodieRecord; -import com.uber.hoodie.common.util.Option; -import com.uber.hoodie.common.util.TypedProperties; -import com.uber.hoodie.common.util.collection.RocksDBBasedMap; -import com.uber.hoodie.exception.HoodieIOException; -import com.uber.hoodie.utilities.schema.SchemaProvider; -import com.uber.hoodie.utilities.sources.config.TestSourceConfig; import java.io.File; import java.io.IOException; import java.util.HashMap; @@ -15,6 +7,14 @@ import java.util.Map; import java.util.stream.Stream; import org.apache.avro.generic.GenericRecord; import org.apache.avro.generic.IndexedRecord; +import org.apache.hudi.common.HoodieTestDataGenerator; +import org.apache.hudi.common.model.HoodieRecord; +import org.apache.hudi.common.util.Option; +import org.apache.hudi.common.util.TypedProperties; +import org.apache.hudi.common.util.collection.RocksDBBasedMap; +import org.apache.hudi.exception.HoodieIOException; +import org.apache.hudi.utilities.schema.SchemaProvider; +import org.apache.hudi.utilities.sources.config.TestSourceConfig; import org.apache.spark.api.java.JavaSparkContext; import org.apache.spark.sql.SparkSession; diff --git a/hoodie-utilities/src/test/java/com/uber/hoodie/utilities/sources/DistributedTestDataSource.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/DistributedTestDataSource.java similarity index 92% rename from hoodie-utilities/src/test/java/com/uber/hoodie/utilities/sources/DistributedTestDataSource.java rename to hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/DistributedTestDataSource.java index 5e42b8a0c..f1ddba446 100644 --- a/hoodie-utilities/src/test/java/com/uber/hoodie/utilities/sources/DistributedTestDataSource.java +++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/DistributedTestDataSource.java @@ -16,16 +16,16 @@ * */ -package com.uber.hoodie.utilities.sources; +package org.apache.hudi.utilities.sources; -import com.uber.hoodie.common.util.Option; -import com.uber.hoodie.common.util.TypedProperties; -import com.uber.hoodie.utilities.schema.SchemaProvider; -import com.uber.hoodie.utilities.sources.config.TestSourceConfig; import java.util.Iterator; import java.util.stream.Collectors; import java.util.stream.IntStream; import org.apache.avro.generic.GenericRecord; +import org.apache.hudi.common.util.Option; +import org.apache.hudi.common.util.TypedProperties; +import org.apache.hudi.utilities.schema.SchemaProvider; +import org.apache.hudi.utilities.sources.config.TestSourceConfig; import org.apache.spark.api.java.JavaRDD; import org.apache.spark.api.java.JavaSparkContext; import org.apache.spark.sql.SparkSession; diff --git a/hoodie-utilities/src/test/java/com/uber/hoodie/utilities/sources/TestDFSSource.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/TestDFSSource.java similarity index 92% rename from hoodie-utilities/src/test/java/com/uber/hoodie/utilities/sources/TestDFSSource.java rename to hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/TestDFSSource.java index 0d561290f..ff51d126c 100644 --- a/hoodie-utilities/src/test/java/com/uber/hoodie/utilities/sources/TestDFSSource.java +++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/TestDFSSource.java @@ -16,20 +16,20 @@ * limitations under the License. */ -package com.uber.hoodie.utilities.sources; +package org.apache.hudi.utilities.sources; import static org.junit.Assert.assertEquals; -import com.uber.hoodie.AvroConversionUtils; -import com.uber.hoodie.common.HoodieTestDataGenerator; -import com.uber.hoodie.common.util.Option; -import com.uber.hoodie.common.util.TypedProperties; -import com.uber.hoodie.utilities.UtilitiesTestBase; -import com.uber.hoodie.utilities.deltastreamer.SourceFormatAdapter; -import com.uber.hoodie.utilities.schema.FilebasedSchemaProvider; import java.io.IOException; import org.apache.avro.generic.GenericRecord; import org.apache.hadoop.fs.Path; +import org.apache.hudi.AvroConversionUtils; +import org.apache.hudi.common.HoodieTestDataGenerator; +import org.apache.hudi.common.util.Option; +import org.apache.hudi.common.util.TypedProperties; +import org.apache.hudi.utilities.UtilitiesTestBase; +import org.apache.hudi.utilities.deltastreamer.SourceFormatAdapter; +import org.apache.hudi.utilities.schema.FilebasedSchemaProvider; import org.apache.spark.api.java.JavaRDD; import org.apache.spark.sql.Dataset; import org.apache.spark.sql.Row; diff --git a/hoodie-utilities/src/test/java/com/uber/hoodie/utilities/sources/TestDataSource.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/TestDataSource.java similarity index 92% rename from hoodie-utilities/src/test/java/com/uber/hoodie/utilities/sources/TestDataSource.java rename to hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/TestDataSource.java index 3b356ac50..61a0c5c51 100644 --- a/hoodie-utilities/src/test/java/com/uber/hoodie/utilities/sources/TestDataSource.java +++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/TestDataSource.java @@ -16,14 +16,14 @@ * limitations under the License. */ -package com.uber.hoodie.utilities.sources; +package org.apache.hudi.utilities.sources; -import com.uber.hoodie.common.util.Option; -import com.uber.hoodie.common.util.TypedProperties; -import com.uber.hoodie.utilities.schema.SchemaProvider; import java.util.List; import java.util.stream.Collectors; import org.apache.avro.generic.GenericRecord; +import org.apache.hudi.common.util.Option; +import org.apache.hudi.common.util.TypedProperties; +import org.apache.hudi.utilities.schema.SchemaProvider; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; import org.apache.spark.api.java.JavaRDD; diff --git a/hoodie-utilities/src/test/java/com/uber/hoodie/utilities/sources/TestKafkaSource.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/TestKafkaSource.java similarity index 94% rename from hoodie-utilities/src/test/java/com/uber/hoodie/utilities/sources/TestKafkaSource.java rename to hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/TestKafkaSource.java index 256a58934..9e31a60ed 100644 --- a/hoodie-utilities/src/test/java/com/uber/hoodie/utilities/sources/TestKafkaSource.java +++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/TestKafkaSource.java @@ -16,22 +16,22 @@ * limitations under the License. */ -package com.uber.hoodie.utilities.sources; +package org.apache.hudi.utilities.sources; import static org.junit.Assert.assertEquals; -import com.uber.hoodie.AvroConversionUtils; -import com.uber.hoodie.common.HoodieTestDataGenerator; -import com.uber.hoodie.common.util.Option; -import com.uber.hoodie.common.util.TypedProperties; -import com.uber.hoodie.utilities.UtilitiesTestBase; -import com.uber.hoodie.utilities.deltastreamer.SourceFormatAdapter; -import com.uber.hoodie.utilities.schema.FilebasedSchemaProvider; -import com.uber.hoodie.utilities.sources.helpers.KafkaOffsetGen.CheckpointUtils; import java.io.IOException; import java.util.HashMap; import kafka.common.TopicAndPartition; import org.apache.avro.generic.GenericRecord; +import org.apache.hudi.AvroConversionUtils; +import org.apache.hudi.common.HoodieTestDataGenerator; +import org.apache.hudi.common.util.Option; +import org.apache.hudi.common.util.TypedProperties; +import org.apache.hudi.utilities.UtilitiesTestBase; +import org.apache.hudi.utilities.deltastreamer.SourceFormatAdapter; +import org.apache.hudi.utilities.schema.FilebasedSchemaProvider; +import org.apache.hudi.utilities.sources.helpers.KafkaOffsetGen.CheckpointUtils; import org.apache.spark.api.java.JavaRDD; import org.apache.spark.sql.Dataset; import org.apache.spark.sql.Row; diff --git a/hoodie-utilities/src/test/java/com/uber/hoodie/utilities/sources/config/TestSourceConfig.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/config/TestSourceConfig.java similarity index 97% rename from hoodie-utilities/src/test/java/com/uber/hoodie/utilities/sources/config/TestSourceConfig.java rename to hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/config/TestSourceConfig.java index 0f6322144..14cb3b636 100644 --- a/hoodie-utilities/src/test/java/com/uber/hoodie/utilities/sources/config/TestSourceConfig.java +++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/config/TestSourceConfig.java @@ -16,7 +16,7 @@ * */ -package com.uber.hoodie.utilities.sources.config; +package org.apache.hudi.utilities.sources.config; /** * Configurations for Test Data Sources diff --git a/hoodie-utilities/src/test/resources/IncrementalPull.sqltemplate b/hudi-utilities/src/test/resources/IncrementalPull.sqltemplate similarity index 100% rename from hoodie-utilities/src/test/resources/IncrementalPull.sqltemplate rename to hudi-utilities/src/test/resources/IncrementalPull.sqltemplate diff --git a/hoodie-utilities/src/test/resources/delta-streamer-config/base.properties b/hudi-utilities/src/test/resources/delta-streamer-config/base.properties similarity index 100% rename from hoodie-utilities/src/test/resources/delta-streamer-config/base.properties rename to hudi-utilities/src/test/resources/delta-streamer-config/base.properties diff --git a/hoodie-utilities/src/test/resources/delta-streamer-config/dfs-source.properties b/hudi-utilities/src/test/resources/delta-streamer-config/dfs-source.properties similarity index 100% rename from hoodie-utilities/src/test/resources/delta-streamer-config/dfs-source.properties rename to hudi-utilities/src/test/resources/delta-streamer-config/dfs-source.properties diff --git a/hoodie-utilities/src/test/resources/delta-streamer-config/kafka-source.properties b/hudi-utilities/src/test/resources/delta-streamer-config/kafka-source.properties similarity index 100% rename from hoodie-utilities/src/test/resources/delta-streamer-config/kafka-source.properties rename to hudi-utilities/src/test/resources/delta-streamer-config/kafka-source.properties diff --git a/hoodie-utilities/src/test/resources/delta-streamer-config/source.avsc b/hudi-utilities/src/test/resources/delta-streamer-config/source.avsc similarity index 100% rename from hoodie-utilities/src/test/resources/delta-streamer-config/source.avsc rename to hudi-utilities/src/test/resources/delta-streamer-config/source.avsc diff --git a/hoodie-utilities/src/test/resources/delta-streamer-config/sql-transformer.properties b/hudi-utilities/src/test/resources/delta-streamer-config/sql-transformer.properties similarity index 100% rename from hoodie-utilities/src/test/resources/delta-streamer-config/sql-transformer.properties rename to hudi-utilities/src/test/resources/delta-streamer-config/sql-transformer.properties diff --git a/hoodie-utilities/src/test/resources/delta-streamer-config/target.avsc b/hudi-utilities/src/test/resources/delta-streamer-config/target.avsc similarity index 100% rename from hoodie-utilities/src/test/resources/delta-streamer-config/target.avsc rename to hudi-utilities/src/test/resources/delta-streamer-config/target.avsc diff --git a/hoodie-utilities/src/test/resources/log4j-surefire-quiet.properties b/hudi-utilities/src/test/resources/log4j-surefire-quiet.properties similarity index 100% rename from hoodie-utilities/src/test/resources/log4j-surefire-quiet.properties rename to hudi-utilities/src/test/resources/log4j-surefire-quiet.properties diff --git a/hoodie-utilities/src/test/resources/log4j-surefire.properties b/hudi-utilities/src/test/resources/log4j-surefire.properties similarity index 100% rename from hoodie-utilities/src/test/resources/log4j-surefire.properties rename to hudi-utilities/src/test/resources/log4j-surefire.properties diff --git a/packaging/hoodie-hadoop-mr-bundle/pom.xml b/packaging/hudi-hadoop-mr-bundle/pom.xml similarity index 87% rename from packaging/hoodie-hadoop-mr-bundle/pom.xml rename to packaging/hudi-hadoop-mr-bundle/pom.xml index b52e12cfd..64a3bbff2 100644 --- a/packaging/hoodie-hadoop-mr-bundle/pom.xml +++ b/packaging/hudi-hadoop-mr-bundle/pom.xml @@ -17,30 +17,30 @@ --> - hoodie - com.uber.hoodie - 0.4.8-SNAPSHOT + hudi + org.apache.hudi + 0.5.0-SNAPSHOT ../../pom.xml 4.0.0 - hoodie-hadoop-mr-bundle + hudi-hadoop-mr-bundle - com.uber.hoodie - hoodie-common + org.apache.hudi + hudi-common ${project.version} - com.uber.hoodie - hoodie-hadoop-mr + org.apache.hudi + hudi-hadoop-mr ${project.version} - - com.uber.hoodie + + org.apache.hudi * @@ -189,50 +189,50 @@ parquet.avro - com.uber.hoodie.parquet.avro + org.apache.hudi.parquet.avro parquet.column - com.uber.hoodie.parquet.column + org.apache.hudi.parquet.column parquet.format. - com.uber.hoodie.parquet.format. + org.apache.hudi.parquet.format. parquet.hadoop. - com.uber.hoodie.parquet.hadoop. + org.apache.hudi.parquet.hadoop. parquet.schema - com.uber.hoodie.parquet.schema + org.apache.hudi.parquet.schema org.apache.commons. - com.uber.hoodie.org.apache.commons. + org.apache.hudi.org.apache.commons. org.apache.commons.codec. - com.uber.hoodie.org.apache.commons.codec. + org.apache.hudi.org.apache.commons.codec. com.esotericsoftware.kryo. - com.uber.hoodie.com.esotericsoftware.kryo. + org.apache.hudi.com.esotericsoftware.kryo. org.objenesis. - com.uber.hoodie.org.objenesis. + org.apache.hudi.org.objenesis. com.esotericsoftware.minlog. - com.uber.hoodie.com.esotericsoftware.minlog. + org.apache.hudi.com.esotericsoftware.minlog. false - com.uber.hoodie:hoodie-common - com.uber.hoodie:hoodie-hadoop-mr + org.apache.hudi:hudi-common + org.apache.hudi:hudi-hadoop-mr com.twitter:parquet-avro com.twitter:parquet-hadoop-bundle com.twitter.common:objectsize diff --git a/packaging/hoodie-presto-bundle/src/main/java/com/uber/hoodie/presto/bundle/Main.java b/packaging/hudi-hadoop-mr-bundle/src/main/java/org/apache/hudi/hadoop/bundle/Main.java similarity index 93% rename from packaging/hoodie-presto-bundle/src/main/java/com/uber/hoodie/presto/bundle/Main.java rename to packaging/hudi-hadoop-mr-bundle/src/main/java/org/apache/hudi/hadoop/bundle/Main.java index e4fb2aa5e..f1c457e11 100644 --- a/packaging/hoodie-presto-bundle/src/main/java/com/uber/hoodie/presto/bundle/Main.java +++ b/packaging/hudi-hadoop-mr-bundle/src/main/java/org/apache/hudi/hadoop/bundle/Main.java @@ -16,9 +16,9 @@ * limitations under the License. */ -package com.uber.hoodie.presto.bundle; +package org.apache.hudi.hadoop.bundle; -import com.uber.hoodie.common.util.ReflectionUtils; +import org.apache.hudi.common.util.ReflectionUtils; /** * A simple main class to dump all classes loaded in current classpath diff --git a/packaging/hoodie-hadoop-mr-bundle/src/main/resources/META-INF/HUDI_LICENSE.txt b/packaging/hudi-hadoop-mr-bundle/src/main/resources/META-INF/HUDI_LICENSE.txt similarity index 100% rename from packaging/hoodie-hadoop-mr-bundle/src/main/resources/META-INF/HUDI_LICENSE.txt rename to packaging/hudi-hadoop-mr-bundle/src/main/resources/META-INF/HUDI_LICENSE.txt diff --git a/packaging/hoodie-hadoop-mr-bundle/src/main/resources/META-INF/HUDI_NOTICE.txt b/packaging/hudi-hadoop-mr-bundle/src/main/resources/META-INF/HUDI_NOTICE.txt similarity index 100% rename from packaging/hoodie-hadoop-mr-bundle/src/main/resources/META-INF/HUDI_NOTICE.txt rename to packaging/hudi-hadoop-mr-bundle/src/main/resources/META-INF/HUDI_NOTICE.txt diff --git a/packaging/hoodie-hive-bundle/pom.xml b/packaging/hudi-hive-bundle/pom.xml similarity index 87% rename from packaging/hoodie-hive-bundle/pom.xml rename to packaging/hudi-hive-bundle/pom.xml index 1dddedc17..4565b39bc 100644 --- a/packaging/hoodie-hive-bundle/pom.xml +++ b/packaging/hudi-hive-bundle/pom.xml @@ -17,14 +17,14 @@ --> - hoodie - com.uber.hoodie - 0.4.8-SNAPSHOT + hudi + org.apache.hudi + 0.5.0-SNAPSHOT ../../pom.xml 4.0.0 - hoodie-hive-bundle + hudi-hive-bundle jar @@ -126,19 +126,19 @@ - com.uber.hoodie - hoodie-hadoop-mr-bundle + org.apache.hudi + hudi-hadoop-mr-bundle ${project.version} - com.uber.hoodie - hoodie-hive + org.apache.hudi + hudi-hive ${project.version} - - com.uber.hoodie + + org.apache.hudi * @@ -171,39 +171,39 @@ com.beust. - com.uber.hoodie.com.beust. + org.apache.hudi.com.beust. org.joda. - com.uber.hoodie.org.joda. + org.apache.hudi.org.joda. com.google. - com.uber.hoodie.com.google. + org.apache.hudi.com.google. org.slf4j. - com.uber.hoodie.org.slf4j. + org.apache.hudi.org.slf4j. org.apache.commons. - com.uber.hoodie.org.apache.commons. + org.apache.hudi.org.apache.commons. parquet.column - com.uber.hoodie.parquet.column + org.apache.hudi.parquet.column parquet.format. - com.uber.hoodie.parquet.format. + org.apache.hudi.parquet.format. parquet.hadoop. - com.uber.hoodie.parquet.hadoop. + org.apache.hudi.parquet.hadoop. parquet.schema. - com.uber.hoodie.parquet.schema. + org.apache.hudi.parquet.schema. false diff --git a/packaging/hoodie-hive-bundle/src/main/java/com/uber/hoodie/hive/bundle/Main.java b/packaging/hudi-hive-bundle/src/main/java/org/apache/hudi/hive/bundle/Main.java similarity index 93% rename from packaging/hoodie-hive-bundle/src/main/java/com/uber/hoodie/hive/bundle/Main.java rename to packaging/hudi-hive-bundle/src/main/java/org/apache/hudi/hive/bundle/Main.java index 1135b484a..65633f954 100644 --- a/packaging/hoodie-hive-bundle/src/main/java/com/uber/hoodie/hive/bundle/Main.java +++ b/packaging/hudi-hive-bundle/src/main/java/org/apache/hudi/hive/bundle/Main.java @@ -16,9 +16,9 @@ * limitations under the License. */ -package com.uber.hoodie.hive.bundle; +package org.apache.hudi.hive.bundle; -import com.uber.hoodie.common.util.ReflectionUtils; +import org.apache.hudi.common.util.ReflectionUtils; /** * A simple main class to dump all classes loaded in current classpath. diff --git a/packaging/hoodie-hive-bundle/src/main/resources/META-INF/HUDI_LICENSE.txt b/packaging/hudi-hive-bundle/src/main/resources/META-INF/HUDI_LICENSE.txt similarity index 100% rename from packaging/hoodie-hive-bundle/src/main/resources/META-INF/HUDI_LICENSE.txt rename to packaging/hudi-hive-bundle/src/main/resources/META-INF/HUDI_LICENSE.txt diff --git a/packaging/hoodie-hive-bundle/src/main/resources/META-INF/HUDI_NOTICE.txt b/packaging/hudi-hive-bundle/src/main/resources/META-INF/HUDI_NOTICE.txt similarity index 100% rename from packaging/hoodie-hive-bundle/src/main/resources/META-INF/HUDI_NOTICE.txt rename to packaging/hudi-hive-bundle/src/main/resources/META-INF/HUDI_NOTICE.txt diff --git a/packaging/hoodie-presto-bundle/pom.xml b/packaging/hudi-presto-bundle/pom.xml similarity index 87% rename from packaging/hoodie-presto-bundle/pom.xml rename to packaging/hudi-presto-bundle/pom.xml index bb49eff6b..b9b3a38fa 100644 --- a/packaging/hoodie-presto-bundle/pom.xml +++ b/packaging/hudi-presto-bundle/pom.xml @@ -17,14 +17,14 @@ --> - hoodie - com.uber.hoodie - 0.4.8-SNAPSHOT + hudi + org.apache.hudi + 0.5.0-SNAPSHOT ../../pom.xml 4.0.0 - hoodie-presto-bundle + hudi-presto-bundle jar @@ -101,8 +101,8 @@ - com.uber.hoodie - hoodie-hadoop-mr-bundle + org.apache.hudi + hudi-hadoop-mr-bundle ${project.version} @@ -133,51 +133,51 @@ com.beust. - com.uber.hoodie.com.beust. + org.apache.hudi.com.beust. org.joda. - com.uber.hoodie.org.joda. + org.apache.hudi.org.joda. com.google. - com.uber.hoodie.com.google. + org.apache.hudi.com.google. org.slf4j. - com.uber.hoodie.org.slf4j. + org.apache.hudi.org.slf4j. org.apache.commons. - com.uber.hoodie.org.apache.commons. + org.apache.hudi.org.apache.commons. parquet.column - com.uber.hoodie.parquet.column + org.apache.hudi.parquet.column parquet.format. - com.uber.hoodie.parquet.format. + org.apache.hudi.parquet.format. parquet.hadoop. - com.uber.hoodie.parquet.hadoop. + org.apache.hudi.parquet.hadoop. parquet.schema. - com.uber.hoodie.parquet.schema. + org.apache.hudi.parquet.schema. com.esotericsoftware.kryo. - com.uber.hoodie.com.esotericsoftware.kryo. + org.apache.hudi.com.esotericsoftware.kryo. org.objenesis. - com.uber.hoodie.org.objenesis. + org.apache.hudi.org.objenesis. com.esotericsoftware.minlog. - com.uber.hoodie.com.esotericsoftware.minlog. + org.apache.hudi.com.esotericsoftware.minlog. false diff --git a/packaging/hoodie-hadoop-mr-bundle/src/main/java/com/uber/hoodie/hadoop/bundle/Main.java b/packaging/hudi-presto-bundle/src/main/java/org/apache/hudi/presto/bundle/Main.java similarity index 93% rename from packaging/hoodie-hadoop-mr-bundle/src/main/java/com/uber/hoodie/hadoop/bundle/Main.java rename to packaging/hudi-presto-bundle/src/main/java/org/apache/hudi/presto/bundle/Main.java index 96927ab81..77329267a 100644 --- a/packaging/hoodie-hadoop-mr-bundle/src/main/java/com/uber/hoodie/hadoop/bundle/Main.java +++ b/packaging/hudi-presto-bundle/src/main/java/org/apache/hudi/presto/bundle/Main.java @@ -16,9 +16,9 @@ * limitations under the License. */ -package com.uber.hoodie.hadoop.bundle; +package org.apache.hudi.presto.bundle; -import com.uber.hoodie.common.util.ReflectionUtils; +import org.apache.hudi.common.util.ReflectionUtils; /** * A simple main class to dump all classes loaded in current classpath diff --git a/packaging/hoodie-presto-bundle/src/main/resources/META-INF/HUDI_LICENSE.txt b/packaging/hudi-presto-bundle/src/main/resources/META-INF/HUDI_LICENSE.txt similarity index 100% rename from packaging/hoodie-presto-bundle/src/main/resources/META-INF/HUDI_LICENSE.txt rename to packaging/hudi-presto-bundle/src/main/resources/META-INF/HUDI_LICENSE.txt diff --git a/packaging/hoodie-presto-bundle/src/main/resources/META-INF/HUDI_NOTICE.txt b/packaging/hudi-presto-bundle/src/main/resources/META-INF/HUDI_NOTICE.txt similarity index 100% rename from packaging/hoodie-presto-bundle/src/main/resources/META-INF/HUDI_NOTICE.txt rename to packaging/hudi-presto-bundle/src/main/resources/META-INF/HUDI_NOTICE.txt diff --git a/packaging/hoodie-spark-bundle/pom.xml b/packaging/hudi-spark-bundle/pom.xml similarity index 85% rename from packaging/hoodie-spark-bundle/pom.xml rename to packaging/hudi-spark-bundle/pom.xml index de553f0f6..304cea40d 100644 --- a/packaging/hoodie-spark-bundle/pom.xml +++ b/packaging/hudi-spark-bundle/pom.xml @@ -17,15 +17,15 @@ --> - hoodie - com.uber.hoodie - 0.4.8-SNAPSHOT + hudi + org.apache.hudi + 0.5.0-SNAPSHOT ../../pom.xml 4.0.0 - com.uber.hoodie - hoodie-spark-bundle + org.apache.hudi + hudi-spark-bundle jar @@ -81,29 +81,30 @@ com.beust. - com.uber.hoodie.com.beust. + org.apache.hudi.com.beust. org.joda. - com.uber.hoodie.org.joda. + org.apache.hudi.org.joda. com.google. - com.uber.hoodie.com.google. + org.apache.hudi.com.google. org.slf4j. - com.uber.hoodie.org.slf4j. + org.apache.hudi.org.slf4j. org.apache. - com.uber.hoodie.org.apache. + org.apache.hudi.org.apache. com.databricks.spark.** org.apache.avro.** org.apache.derby.** org.apache.hadoop.** org.apache.hive.** + org.apache.hudi.** org.apache.logging.log4j.** org.apache.log4j.** org.apache.spark.** @@ -112,59 +113,59 @@ parquet.column - com.uber.hoodie.parquet.column + org.apache.hudi.parquet.column parquet.format. - com.uber.hoodie.parquet.format. + org.apache.hudi.parquet.format. parquet.hadoop. - com.uber.hoodie.parquet.hadoop. + org.apache.hudi.parquet.hadoop. parquet.schema - com.uber.hoodie.parquet.schema + org.apache.hudi.parquet.schema org.apache.hive.jdbc. - com.uber.hoodie.org.apache.hive.jdbc. + org.apache.hudi.org.apache.hive.jdbc. org.apache.hadoop.hive.metastore. - com.uber.hoodie.org.apache.hadoop_hive.metastore. + org.apache.hudi.org.apache.hadoop_hive.metastore. org.apache.hive.common. - com.uber.hoodie.org.apache.hive.common. + org.apache.hudi.org.apache.hive.common. org.apache.hadoop.hive.common. - com.uber.hoodie.org.apache.hadoop_hive.common. + org.apache.hudi.org.apache.hadoop_hive.common. org.apache.hadoop.hive.conf. - com.uber.hoodie.org.apache.hadoop_hive.conf. + org.apache.hudi.org.apache.hadoop_hive.conf. org.apache.hive.service. - com.uber.hoodie.org.apache.hive.service. + org.apache.hudi.org.apache.hive.service. org.apache.hadoop.hive.service. - com.uber.hoodie.org.apache.hadoop_hive.service. + org.apache.hudi.org.apache.hadoop_hive.service. com.esotericsoftware.kryo. - com.uber.hoodie.com.esotericsoftware.kryo. + org.apache.hudi.com.esotericsoftware.kryo. org.objenesis. - com.uber.hoodie.org.objenesis. + org.apache.hudi.org.objenesis. com.esotericsoftware.minlog. - com.uber.hoodie.com.esotericsoftware.minlog. + org.apache.hudi.com.esotericsoftware.minlog. false @@ -303,28 +304,28 @@ commons-configuration2 - com.uber.hoodie - hoodie-common + org.apache.hudi + hudi-common ${project.version} - com.uber.hoodie - hoodie-hadoop-mr + org.apache.hudi + hudi-hadoop-mr ${project.version} - com.uber.hoodie - hoodie-hive + org.apache.hudi + hudi-hive ${project.version} - com.uber.hoodie - hoodie-client + org.apache.hudi + hudi-client ${project.version} - com.uber.hoodie - hoodie-spark + org.apache.hudi + hudi-spark ${project.version} diff --git a/packaging/hoodie-spark-bundle/src/main/java/com/uber/hoodie/spark/bundle/Main.java b/packaging/hudi-spark-bundle/src/main/java/org/apache/hudi/spark/bundle/Main.java similarity index 93% rename from packaging/hoodie-spark-bundle/src/main/java/com/uber/hoodie/spark/bundle/Main.java rename to packaging/hudi-spark-bundle/src/main/java/org/apache/hudi/spark/bundle/Main.java index d9f63043f..1489bf5ba 100644 --- a/packaging/hoodie-spark-bundle/src/main/java/com/uber/hoodie/spark/bundle/Main.java +++ b/packaging/hudi-spark-bundle/src/main/java/org/apache/hudi/spark/bundle/Main.java @@ -16,9 +16,9 @@ * limitations under the License. */ -package com.uber.hoodie.spark.bundle; +package org.apache.hudi.spark.bundle; -import com.uber.hoodie.common.util.ReflectionUtils; +import org.apache.hudi.common.util.ReflectionUtils; /** * A simple main class to dump all classes loaded in current classpath diff --git a/packaging/hoodie-spark-bundle/src/main/resources/META-INF/HUDI_LICENSE.txt b/packaging/hudi-spark-bundle/src/main/resources/META-INF/HUDI_LICENSE.txt similarity index 100% rename from packaging/hoodie-spark-bundle/src/main/resources/META-INF/HUDI_LICENSE.txt rename to packaging/hudi-spark-bundle/src/main/resources/META-INF/HUDI_LICENSE.txt diff --git a/packaging/hoodie-spark-bundle/src/main/resources/META-INF/HUDI_NOTICE.txt b/packaging/hudi-spark-bundle/src/main/resources/META-INF/HUDI_NOTICE.txt similarity index 100% rename from packaging/hoodie-spark-bundle/src/main/resources/META-INF/HUDI_NOTICE.txt rename to packaging/hudi-spark-bundle/src/main/resources/META-INF/HUDI_NOTICE.txt diff --git a/packaging/hoodie-utilities-bundle/pom.xml b/packaging/hudi-utilities-bundle/pom.xml similarity index 85% rename from packaging/hoodie-utilities-bundle/pom.xml rename to packaging/hudi-utilities-bundle/pom.xml index 73f50bfd0..c8dc51890 100644 --- a/packaging/hoodie-utilities-bundle/pom.xml +++ b/packaging/hudi-utilities-bundle/pom.xml @@ -18,13 +18,13 @@ - hoodie - com.uber.hoodie - 0.4.8-SNAPSHOT + hudi + org.apache.hudi + 0.5.0-SNAPSHOT ../../pom.xml 4.0.0 - hoodie-utilities-bundle + hudi-utilities-bundle jar @@ -69,13 +69,13 @@ commons-dbcp:commons-dbcp commons-lang:commons-lang commons-pool:commons-pool - com.uber.hoodie:hoodie-common - com.uber.hoodie:hoodie-client - com.uber.hoodie:hoodie-utilities - com.uber.hoodie:hoodie-spark - com.uber.hoodie:hoodie-hive - com.uber.hoodie:hoodie-hadoop-mr - com.uber.hoodie:hoodie-timeline-service + org.apache.hudi:hudi-common + org.apache.hudi:hudi-client + org.apache.hudi:hudi-utilities + org.apache.hudi:hudi-spark + org.apache.hudi:hudi-hive + org.apache.hudi:hudi-hadoop-mr + org.apache.hudi:hudi-timeline-service com.beust:jcommander com.twitter:bijection-avro_2.11 com.twitter:bijection-core_2.11 @@ -105,67 +105,67 @@ com.beust.jcommander. - com.uber.hoodie.com.beust.jcommander. + org.apache.hudi.com.beust.jcommander. org.apache.commons.dbcp. - com.uber.hoodie.org.apache.commons.dbcp. + org.apache.hudi.org.apache.commons.dbcp. org.apache.commons.lang. - com.uber.hoodie.org.apache.commons.lang. + org.apache.hudi.org.apache.commons.lang. org.apache.commons.pool. - com.uber.hoodie.org.apache.commons.pool. + org.apache.hudi.org.apache.commons.pool. org.apache.hive.jdbc. - com.uber.hoodie.org.apache.hive.jdbc. + org.apache.hudi.org.apache.hive.jdbc. org.apache.hadoop.hive.metastore. - com.uber.hoodie.org.apache.hadoop_hive.metastore. + org.apache.hudi.org.apache.hadoop_hive.metastore. org.apache.hive.common. - com.uber.hoodie.org.apache.hive.common. + org.apache.hudi.org.apache.hive.common. org.apache.hadoop.hive.common. - com.uber.hoodie.org.apache.hadoop_hive.common. + org.apache.hudi.org.apache.hadoop_hive.common. org.apache.hadoop.hive.conf. - com.uber.hoodie.org.apache.hadoop_hive.conf. + org.apache.hudi.org.apache.hadoop_hive.conf. org.apache.hive.service. - com.uber.hoodie.org.apache.hive.service. + org.apache.hudi.org.apache.hive.service. org.apache.hadoop.hive.service. - com.uber.hoodie.org.apache.hadoop_hive.service. + org.apache.hudi.org.apache.hadoop_hive.service. com.esotericsoftware.kryo. - com.uber.hoodie.com.esotericsoftware.kryo. + org.apache.hudi.com.esotericsoftware.kryo. org.objenesis. - com.uber.hoodie.org.objenesis. + org.apache.hudi.org.objenesis. com.esotericsoftware.minlog. - com.uber.hoodie.com.esotericsoftware.minlog. + org.apache.hudi.com.esotericsoftware.minlog. com.codahale.metrics. - com.uber.hoodie.com.codahale.metrics. + org.apache.hudi.com.codahale.metrics. org.apache.commons.codec. - com.uber.hoodie.org.apache.commons.codec. + org.apache.hudi.org.apache.commons.codec. @@ -222,14 +222,14 @@ - com.uber.hoodie - hoodie-common + org.apache.hudi + hudi-common ${project.version} - com.uber.hoodie - hoodie-common + org.apache.hudi + hudi-common ${project.version} tests test-jar @@ -237,8 +237,8 @@ - com.uber.hoodie - hoodie-hive + org.apache.hudi + hudi-hive ${project.version} tests test-jar @@ -246,8 +246,8 @@ - com.uber.hoodie - hoodie-spark + org.apache.hudi + hudi-spark ${project.version} @@ -301,8 +301,8 @@ - com.uber.hoodie - hoodie-hive + org.apache.hudi + hudi-hive ${project.version} @@ -313,20 +313,20 @@ - com.uber.hoodie - hoodie-client + org.apache.hudi + hudi-client ${project.version} - com.uber.hoodie - hoodie-utilities + org.apache.hudi + hudi-utilities ${project.version} - com.uber.hoodie - hoodie-client + org.apache.hudi + hudi-client ${project.version} tests test-jar diff --git a/packaging/hoodie-utilities-bundle/src/main/java/com/uber/hoodie/utilities/bundle/Main.java b/packaging/hudi-utilities-bundle/src/main/java/org/apache/hudi/utilities/bundle/Main.java similarity index 93% rename from packaging/hoodie-utilities-bundle/src/main/java/com/uber/hoodie/utilities/bundle/Main.java rename to packaging/hudi-utilities-bundle/src/main/java/org/apache/hudi/utilities/bundle/Main.java index dfc95ea8c..a4e2dcb34 100644 --- a/packaging/hoodie-utilities-bundle/src/main/java/com/uber/hoodie/utilities/bundle/Main.java +++ b/packaging/hudi-utilities-bundle/src/main/java/org/apache/hudi/utilities/bundle/Main.java @@ -16,9 +16,9 @@ * limitations under the License. */ -package com.uber.hoodie.utilities.bundle; +package org.apache.hudi.utilities.bundle; -import com.uber.hoodie.common.util.ReflectionUtils; +import org.apache.hudi.common.util.ReflectionUtils; /** * A simple main class to dump all classes loaded in current classpath diff --git a/packaging/hoodie-utilities-bundle/src/main/resources/META-INF/HUDI_LICENSE.txt b/packaging/hudi-utilities-bundle/src/main/resources/META-INF/HUDI_LICENSE.txt similarity index 100% rename from packaging/hoodie-utilities-bundle/src/main/resources/META-INF/HUDI_LICENSE.txt rename to packaging/hudi-utilities-bundle/src/main/resources/META-INF/HUDI_LICENSE.txt diff --git a/packaging/hoodie-utilities-bundle/src/main/resources/META-INF/HUDI_NOTICE.txt b/packaging/hudi-utilities-bundle/src/main/resources/META-INF/HUDI_NOTICE.txt similarity index 100% rename from packaging/hoodie-utilities-bundle/src/main/resources/META-INF/HUDI_NOTICE.txt rename to packaging/hudi-utilities-bundle/src/main/resources/META-INF/HUDI_NOTICE.txt diff --git a/pom.xml b/pom.xml index bebd4248e..5671d11b8 100644 --- a/pom.xml +++ b/pom.xml @@ -20,10 +20,10 @@ 4.0.0 - com.uber.hoodie - hoodie + org.apache.hudi + hudi pom - 0.4.8-SNAPSHOT + 0.5.0-SNAPSHOT Hoodie is a Apache Spark library that provides the ability to efficiently do incremental processing on datasets in HDFS @@ -31,21 +31,21 @@ Hoodie - hoodie-common - hoodie-cli - hoodie-client - hoodie-hadoop-mr - hoodie-hive - hoodie-spark - hoodie-timeline-service - hoodie-utilities - packaging/hoodie-hadoop-mr-bundle - packaging/hoodie-hive-bundle - packaging/hoodie-spark-bundle - packaging/hoodie-presto-bundle - packaging/hoodie-utilities-bundle + hudi-common + hudi-cli + hudi-client + hudi-hadoop-mr + hudi-hive + hudi-spark + hudi-timeline-service + hudi-utilities + packaging/hudi-hadoop-mr-bundle + packaging/hudi-hive-bundle + packaging/hudi-spark-bundle + packaging/hudi-presto-bundle + packaging/hudi-utilities-bundle docker/hoodie/hadoop - hoodie-integ-test + hudi-integ-test @@ -151,22 +151,22 @@ - scm:git:git@github.com:uber/hoodie.git - scm:git:git@github.com:uber/hoodie.git - git@github.com:uber/hoodie.git + scm:git:git@github.com:uber/hudi.git + scm:git:git@github.com:uber/hudi.git + git@github.com:uber/hudi.git HEAD User List - hoodie-user@googlegroups.com - https://groups.google.com/d/forum/hoodie-user/ + hudi-user@googlegroups.com + https://groups.google.com/d/forum/hudi-user/ Developer List - hoodie-dev@googlegroups.com - https://groups.google.com/d/forum/hoodie-dev/ + hudi-dev@googlegroups.com + https://groups.google.com/d/forum/hudi-dev/ @@ -479,7 +479,7 @@ - + joda-time joda-time ${joda.version} @@ -1069,6 +1069,6 @@ GitHub - https://github.com/uber/hoodie/issues + https://github.org/apache/hudi/issues