diff --git a/hudi-common/src/test/java/org/apache/hudi/common/testutils/HoodieTestDataGenerator.java b/hudi-common/src/test/java/org/apache/hudi/common/testutils/HoodieTestDataGenerator.java
index e23d88586..ed7a4582e 100644
--- a/hudi-common/src/test/java/org/apache/hudi/common/testutils/HoodieTestDataGenerator.java
+++ b/hudi-common/src/test/java/org/apache/hudi/common/testutils/HoodieTestDataGenerator.java
@@ -68,7 +68,6 @@ import java.util.stream.Collectors;
import java.util.stream.IntStream;
import java.util.stream.Stream;
-
/**
* Class to be used in tests to keep generating test inserts and updates against a corpus.
*
@@ -98,7 +97,8 @@ public class HoodieTestDataGenerator {
+ "{\"name\": \"amount\",\"type\": \"double\"},{\"name\": \"currency\", \"type\": \"string\"}]}},";
public static final String FARE_FLATTENED_SCHEMA = "{\"name\": \"fare\", \"type\": \"double\"},"
+ "{\"name\": \"currency\", \"type\": \"string\"},";
- public static final String TIP_NESTED_SCHEMA = "{\"name\": \"tip_history\", \"type\": {\"type\": \"array\", \"items\": {\"type\": \"record\", \"name\": \"tip_history\", \"fields\": ["
+ public static final String TIP_NESTED_SCHEMA = "{\"name\": \"tip_history\", \"default\": null, \"type\": {\"type\": "
+ + "\"array\", \"items\": {\"type\": \"record\", \"default\": null, \"name\": \"tip_history\", \"fields\": ["
+ "{\"name\": \"amount\", \"type\": \"double\"}, {\"name\": \"currency\", \"type\": \"string\"}]}}},";
public static final String MAP_TYPE_SCHEMA = "{\"name\": \"city_to_state\", \"type\": {\"type\": \"map\", \"values\": \"string\"}},";
public static final String EXTRA_TYPE_SCHEMA = "{\"name\": \"distance_in_meters\", \"type\": \"int\"},"
@@ -254,7 +254,6 @@ public class HoodieTestDataGenerator {
rec.put("begin_lon", RAND.nextDouble());
rec.put("end_lat", RAND.nextDouble());
rec.put("end_lon", RAND.nextDouble());
-
if (isFlattened) {
rec.put("fare", RAND.nextDouble() * 100);
rec.put("currency", "USD");
@@ -730,7 +729,7 @@ public class HoodieTestDataGenerator {
public boolean deleteExistingKeyIfPresent(HoodieKey key) {
Map existingKeys = existingKeysBySchema.get(TRIP_EXAMPLE_SCHEMA);
Integer numExistingKeys = numKeysBySchema.get(TRIP_EXAMPLE_SCHEMA);
- for (Map.Entry entry: existingKeys.entrySet()) {
+ for (Map.Entry entry : existingKeys.entrySet()) {
if (entry.getValue().key.equals(key)) {
int index = entry.getKey();
existingKeys.put(index, existingKeys.get(numExistingKeys - 1));
@@ -740,10 +739,18 @@ public class HoodieTestDataGenerator {
return true;
}
}
-
return false;
}
+ public List generateGenericRecords(int numRecords) {
+ List list = new ArrayList<>();
+ IntStream.range(0, numRecords).forEach(i -> {
+ list.add(generateGenericRecord(UUID.randomUUID().toString(), UUID.randomUUID().toString(), UUID.randomUUID()
+ .toString(), RAND.nextDouble()));
+ });
+ return list;
+ }
+
public String[] getPartitionPaths() {
return partitionPaths;
}
diff --git a/hudi-hive-sync/src/main/java/org/apache/hudi/hive/HiveSyncTool.java b/hudi-hive-sync/src/main/java/org/apache/hudi/hive/HiveSyncTool.java
index 877ba4791..d8292526e 100644
--- a/hudi-hive-sync/src/main/java/org/apache/hudi/hive/HiveSyncTool.java
+++ b/hudi-hive-sync/src/main/java/org/apache/hudi/hive/HiveSyncTool.java
@@ -112,6 +112,15 @@ public class HiveSyncTool {
+ " of type " + hoodieHiveClient.getTableType());
// Check if the necessary table exists
boolean tableExists = hoodieHiveClient.doesTableExist(tableName);
+
+ // check if the database exists else create it
+ try {
+ hoodieHiveClient.updateHiveSQL("create database if not exists " + cfg.databaseName);
+ } catch (Exception e) {
+ // this is harmless since table creation will fail anyways, creation of DB is needed for in-memory testing
+ LOG.warn("Unable to create database", e);
+ }
+
// Get the parquet schema for this table looking at the latest commit
MessageType schema = hoodieHiveClient.getDataSchema();
// Sync schema if needed
diff --git a/hudi-hive-sync/src/main/java/org/apache/hudi/hive/HoodieHiveClient.java b/hudi-hive-sync/src/main/java/org/apache/hudi/hive/HoodieHiveClient.java
index f1034e3cd..2bb3fd112 100644
--- a/hudi-hive-sync/src/main/java/org/apache/hudi/hive/HoodieHiveClient.java
+++ b/hudi-hive-sync/src/main/java/org/apache/hudi/hive/HoodieHiveClient.java
@@ -572,4 +572,8 @@ public class HoodieHiveClient {
return new PartitionEvent(PartitionEventType.UPDATE, storagePartition);
}
}
+
+ public IMetaStoreClient getClient() {
+ return client;
+ }
}
\ No newline at end of file
diff --git a/hudi-hive-sync/src/test/java/org/apache/hudi/hive/testutils/HiveTestService.java b/hudi-hive-sync/src/test/java/org/apache/hudi/hive/testutils/HiveTestService.java
index 2f9880304..51a5bf5ed 100644
--- a/hudi-hive-sync/src/test/java/org/apache/hudi/hive/testutils/HiveTestService.java
+++ b/hudi-hive-sync/src/test/java/org/apache/hudi/hive/testutils/HiveTestService.java
@@ -126,10 +126,21 @@ public class HiveTestService {
public void stop() {
resetSystemProperties();
if (tServer != null) {
- tServer.stop();
+ try {
+ tServer.stop();
+ } catch (Exception e) {
+ LOG.error("Stop meta store failed", e);
+ }
}
if (hiveServer != null) {
- hiveServer.stop();
+ try {
+ hiveServer.stop();
+ } catch (Exception e) {
+ LOG.error("Stop hive server failed", e);
+ }
+ }
+ if (executorService != null) {
+ executorService.shutdownNow();
}
LOG.info("Hive Minicluster service shut down.");
tServer = null;
diff --git a/hudi-integ-test/README.md b/hudi-integ-test/README.md
new file mode 100644
index 000000000..d87fec3ad
--- /dev/null
+++ b/hudi-integ-test/README.md
@@ -0,0 +1,262 @@
+
+
+This page describes in detail how to run end to end tests on a hudi dataset that helps in improving our confidence
+in a release as well as perform large scale performance benchmarks.
+
+# Objectives
+
+1. Test with different versions of core libraries and components such as `hdfs`, `parquet`, `spark`,
+`hive` and `avro`.
+2. Generate different types of workloads across different dimensions such as `payload size`, `number of updates`,
+`number of inserts`, `number of partitions`
+3. Perform multiple types of operations such as `insert`, `bulk_insert`, `upsert`, `compact`, `query`
+4. Support custom post process actions and validations
+
+# High Level Design
+
+The Hudi test suite runs as a long running spark job. The suite is divided into the following high level components :
+
+## Workload Generation
+
+This component does the work of generating the workload; `inserts`, `upserts` etc.
+
+## Workload Scheduling
+
+Depending on the type of workload generated, data is either ingested into the target hudi
+dataset or the corresponding workload operation is executed. For example compaction does not necessarily need a workload
+to be generated/ingested but can require an execution.
+
+## Other actions/operatons
+
+The test suite supports different types of operations besides ingestion such as Hive Query execution, Clean action etc.
+
+# Usage instructions
+
+
+## Entry class to the test suite
+
+```
+org.apache.hudi.testsuite.HoodieTestSuiteJob.java - Entry Point of the hudi test suite job. This
+class wraps all the functionalities required to run a configurable integration suite.
+```
+
+## Configurations required to run the job
+```
+org.apache.hudi.testsuite.HoodieTestSuiteJob.HoodieTestSuiteConfig - Config class that drives the behavior of the
+integration test suite. This class extends from com.uber.hoodie.utilities.DeltaStreamerConfig. Look at
+link#HudiDeltaStreamer page to learn about all the available configs applicable to your test suite.
+```
+
+## Generating a custom Workload Pattern
+
+There are 2 ways to generate a workload pattern
+
+ 1.Programatically
+
+Choose to write up the entire DAG of operations programatically, take a look at `WorkflowDagGenerator` class.
+Once you're ready with the DAG you want to execute, simply pass the class name as follows:
+
+```
+spark-submit
+...
+...
+--class org.apache.hudi.testsuite.HoodieTestSuiteJob
+--workload-generator-classname org.apache.hudi.testsuite.dag.scheduler.
+...
+```
+
+ 2.YAML file
+
+Choose to write up the entire DAG of operations in YAML, take a look at `complex-workload-dag-cow.yaml` or
+`complex-workload-dag-mor.yaml`.
+Once you're ready with the DAG you want to execute, simply pass the yaml file path as follows:
+
+```
+spark-submit
+...
+...
+--class org.apache.hudi.testsuite.HoodieTestSuiteJob
+--workload-yaml-path /path/to/your-workflow-dag.yaml
+...
+```
+
+## Building the test suite
+
+The test suite can be found in the `hudi-integ-test` module. Use the `prepare_integration_suite.sh` script to
+build
+the test suite, you can provide different parameters to the script.
+
+```
+shell$ ./prepare_integration_suite.sh --help
+Usage: prepare_integration_suite.sh
+ --spark-command, prints the spark command
+ -h, hdfs-version
+ -s, spark version
+ -p, parquet version
+ -a, avro version
+ -s, hive version
+```
+
+```
+shell$ ./prepare_integration_suite.sh
+....
+....
+Final command : mvn clean install -DskipTests
+```
+
+## Running on the cluster or in your local machine
+Copy over the necessary files and jars that are required to your cluster and then run the following spark-submit
+command after replacing the correct values for the parameters.
+NOTE : The properties-file should have all the necessary information required to ingest into a Hudi dataset. For more
+ information on what properties need to be set, take a look at the test suite section under demo steps.
+```
+shell$ ./prepare_integration_suite.sh --spark-command
+spark-submit --packages com.databricks:spark-avro_2.11:4.0.0 --master prepare_integration_suite.sh --deploy-mode
+--properties-file --class org.apache.hudi.integ.testsuite.HoodieTestSuiteJob target/hudi-integ-test-0.6
+.0-SNAPSHOT.jar --source-class --source-ordering-field --input-base-path --target-base-path --target-table --props --storage-type --payload-class --workload-yaml-path --input-file-size --
+```
+
+## Running through a test-case (local)
+Take a look at the `TestHoodieTestSuiteJob` to check how you can run the entire suite using JUnit.
+
+## Running an end to end test suite in Local Docker environment
+
+```
+docker exec -it adhoc-2 /bin/bash
+# COPY_ON_WRITE tables
+=========================
+## Run the following command to start the test suite
+spark-submit \
+--packages com.databricks:spark-avro_2.11:4.0.0 \
+--conf spark.task.cpus=1 \
+--conf spark.executor.cores=1 \
+--conf spark.task.maxFailures=100 \
+--conf spark.memory.fraction=0.4 \
+--conf spark.rdd.compress=true \
+--conf spark.kryoserializer.buffer.max=2000m \
+--conf spark.serializer=org.apache.spark.serializer.KryoSerializer \
+--conf spark.memory.storageFraction=0.1 \
+--conf spark.shuffle.service.enabled=true \
+--conf spark.sql.hive.convertMetastoreParquet=false \
+--conf spark.ui.port=5555 \
+--conf spark.driver.maxResultSize=12g \
+--conf spark.executor.heartbeatInterval=120s \
+--conf spark.network.timeout=600s \
+--conf spark.eventLog.overwrite=true \
+--conf spark.eventLog.enabled=true \
+--conf spark.yarn.max.executor.failures=10 \
+--conf spark.sql.catalogImplementation=hive \
+--conf spark.sql.shuffle.partitions=1000 \
+--class org.apache.hudi.testsuite.HoodieTestSuiteJob $HUDI_TEST_SUITE_BUNDLE \
+--source-ordering-field timestamp \
+--target-base-path /user/hive/warehouse/hudi-integ-test-suite/output \
+--input-base-path /user/hive/warehouse/hudi-integ-test-suite/input \
+--target-table test_table \
+--props /var/hoodie/ws/docker/demo/config/testsuite/test-source.properties \
+--schemaprovider-class org.apache.hudi.utilities.schema.FilebasedSchemaProvider \
+--source-limit 300000 \
+--source-class org.apache.hudi.utilities.sources.AvroDFSSource \
+--input-file-size 125829120 \
+--workload-yaml-path /var/hoodie/ws/docker/demo/config/testsuite/complex-workflow-dag-cow.yaml \
+--storage-type COPY_ON_WRITE \
+--compact-scheduling-minshare 1 \
+--hoodie-conf "hoodie.deltastreamer.source.test.num_partitions=100" \
+--hoodie-conf "hoodie.deltastreamer.source.test.datagen.use_rocksdb_for_storing_existing_keys=false" \
+--hoodie-conf "hoodie.deltastreamer.source.test.max_unique_records=100000000" \
+--hoodie-conf "hoodie.embed.timeline.server=false" \
+--hoodie-conf "hoodie.datasource.write.recordkey.field=_row_key" \
+--hoodie-conf "hoodie.deltastreamer.source.dfs.root=/user/hive/warehouse/hudi-integ-test-suite/input" \
+--hoodie-conf "hoodie.datasource.write.keygenerator.class=org.apache.hudi.ComplexKeyGenerator" \
+--hoodie-conf "hoodie.datasource.write.partitionpath.field=timestamp" \
+--hoodie-conf "hoodie.deltastreamer.schemaprovider.source.schema.file=/var/hoodie/ws/docker/demo/config/testsuite/source.avsc" \
+--hoodie-conf "hoodie.datasource.hive_sync.assume_date_partitioning=false" \
+--hoodie-conf "hoodie.datasource.hive_sync.jdbcurl=jdbc:hive2://hiveserver:10000/" \
+--hoodie-conf "hoodie.datasource.hive_sync.database=testdb" \
+--hoodie-conf "hoodie.datasource.hive_sync.table=test_table" \
+--hoodie-conf "hoodie.datasource.hive_sync.partition_extractor_class=org.apache.hudi.hive.NonPartitionedExtractor" \
+--hoodie-conf "hoodie.datasource.hive_sync.assume_date_partitioning=true" \
+--hoodie-conf "hoodie.datasource.write.keytranslator.class=org.apache.hudi.DayBasedPartitionPathKeyTranslator" \
+--hoodie-conf "hoodie.deltastreamer.schemaprovider.target.schema.file=/var/hoodie/ws/docker/demo/config/testsuite/source.avsc"
+...
+...
+2019-11-03 05:44:47 INFO DagScheduler:69 - ----------- Finished workloads ----------
+2019-11-03 05:44:47 INFO HoodieTestSuiteJob:138 - Finished scheduling all tasks
+...
+2019-11-03 05:44:48 INFO SparkContext:54 - Successfully stopped SparkContext
+# MERGE_ON_READ tables
+=========================
+## Run the following command to start the test suite
+spark-submit \
+--packages com.databricks:spark-avro_2.11:4.0.0 \
+--conf spark.task.cpus=1 \
+--conf spark.executor.cores=1 \
+--conf spark.task.maxFailures=100 \
+--conf spark.memory.fraction=0.4 \
+--conf spark.rdd.compress=true \
+--conf spark.kryoserializer.buffer.max=2000m \
+--conf spark.serializer=org.apache.spark.serializer.KryoSerializer \
+--conf spark.memory.storageFraction=0.1 \
+--conf spark.shuffle.service.enabled=true \
+--conf spark.sql.hive.convertMetastoreParquet=false \
+--conf spark.ui.port=5555 \
+--conf spark.driver.maxResultSize=12g \
+--conf spark.executor.heartbeatInterval=120s \
+--conf spark.network.timeout=600s \
+--conf spark.eventLog.overwrite=true \
+--conf spark.eventLog.enabled=true \
+--conf spark.yarn.max.executor.failures=10 \
+--conf spark.sql.catalogImplementation=hive \
+--conf spark.sql.shuffle.partitions=1000 \
+--class org.apache.hudi.testsuite.HoodieTestSuiteJob $HUDI_TEST_SUITE_BUNDLE \
+--source-ordering-field timestamp \
+--target-base-path /user/hive/warehouse/hudi-integ-test-suite/output \
+--input-base-path /user/hive/warehouse/hudi-integ-test-suite/input \
+--target-table test_table \
+--props /var/hoodie/ws/docker/demo/config/testsuite/test-source.properties \
+--schemaprovider-class org.apache.hudi.utilities.schema.FilebasedSchemaProvider \
+--source-limit 300000 \
+--source-class org.apache.hudi.utilities.sources.AvroDFSSource \
+--input-file-size 125829120 \
+--workload-yaml-path /var/hoodie/ws/docker/demo/config/testsuite/complex-workflow-dag-mor.yaml \
+--storage-type MERGE_ON_READ \
+--compact-scheduling-minshare 1 \
+--hoodie-conf "hoodie.deltastreamer.source.test.num_partitions=100" \
+--hoodie-conf "hoodie.deltastreamer.source.test.datagen.use_rocksdb_for_storing_existing_keys=false" \
+--hoodie-conf "hoodie.deltastreamer.source.test.max_unique_records=100000000" \
+--hoodie-conf "hoodie.embed.timeline.server=false" \
+--hoodie-conf "hoodie.datasource.write.recordkey.field=_row_key" \
+--hoodie-conf "hoodie.deltastreamer.source.dfs.root=/user/hive/warehouse/hudi-integ-test-suite/input" \
+--hoodie-conf "hoodie.datasource.write.keygenerator.class=org.apache.hudi.ComplexKeyGenerator" \
+--hoodie-conf "hoodie.datasource.write.partitionpath.field=timestamp" \
+--hoodie-conf "hoodie.deltastreamer.schemaprovider.source.schema.file=/var/hoodie/ws/docker/demo/config/testsuite/source.avsc" \
+--hoodie-conf "hoodie.datasource.hive_sync.assume_date_partitioning=false" \
+--hoodie-conf "hoodie.datasource.hive_sync.jdbcurl=jdbc:hive2://hiveserver:10000/" \
+--hoodie-conf "hoodie.datasource.hive_sync.database=testdb" \
+--hoodie-conf "hoodie.datasource.hive_sync.table=test_table" \
+--hoodie-conf "hoodie.datasource.hive_sync.partition_extractor_class=org.apache.hudi.hive.NonPartitionedExtractor" \
+--hoodie-conf "hoodie.datasource.hive_sync.assume_date_partitioning=true" \
+--hoodie-conf "hoodie.datasource.write.keytranslator.class=org.apache.hudi.DayBasedPartitionPathKeyTranslator" \
+--hoodie-conf "hoodie.deltastreamer.schemaprovider.target.schema.file=/var/hoodie/ws/docker/demo/config/testsuite/source.avsc"
+...
+...
+2019-11-03 05:44:47 INFO DagScheduler:69 - ----------- Finished workloads ----------
+2019-11-03 05:44:47 INFO HoodieTestSuiteJob:138 - Finished scheduling all tasks
+...
+2019-11-03 05:44:48 INFO SparkContext:54 - Successfully stopped SparkContext
+```
+
\ No newline at end of file
diff --git a/hudi-integ-test/pom.xml b/hudi-integ-test/pom.xml
index b9bcc6d96..e81f335ba 100644
--- a/hudi-integ-test/pom.xml
+++ b/hudi-integ-test/pom.xml
@@ -51,12 +51,69 @@
test
+
+
+
+ org.eclipse.jetty.aggregate
+ jetty-all
+ ${jetty.version}
+ uber
+ test
+
+
+
+ org.apache.spark
+ spark-sql_${scala.binary.version}
+
+
+ org.mortbay.jetty
+ *
+
+
+ javax.servlet.jsp
+ *
+
+
+ javax.servlet
+ *
+
+
+ org.eclipse.jetty
+ *
+
+
+
+
org.apache.hudi
hudi-spark_${scala.binary.version}
${project.version}
+
+ org.apache.hudi
+ hudi-utilities_${scala.binary.version}
+ ${project.version}
+ provided
+
+
+ org.mortbay.jetty
+ *
+
+
+ javax.servlet.jsp
+ *
+
+
+ javax.servlet
+ *
+
+
+ com.databricks
+ spark-avro_${scala.binary.version}
+
+
+
@@ -89,6 +146,67 @@
test-jar
test
+
+ org.apache.hudi
+ hudi-hive-sync
+ ${project.version}
+
+
+ org.mortbay.jetty
+ *
+
+
+ javax.servlet.jsp
+ *
+
+
+ javax.servlet
+ *
+
+
+ tests
+ test-jar
+
+
+ org.apache.hudi
+ hudi-utilities_${scala.binary.version}
+ ${project.version}
+ tests
+ test-jar
+ test
+
+
+ org.mortbay.jetty
+ *
+
+
+ javax.servlet.jsp
+ *
+
+
+ javax.servlet
+ *
+
+
+
+
+ org.apache.hudi
+ hudi-client
+ ${project.version}
+ test-jar
+
+
+
+
+ com.fasterxml.jackson.dataformat
+ jackson-dataformat-yaml
+ 2.7.4
+
+
+ com.fasterxml.jackson.core
+ jackson-databind
+ 2.6.7.3
+
@@ -107,6 +225,87 @@
test
+
+
+
+ org.apache.hadoop
+ hadoop-common
+ tests
+
+
+ org.mortbay.jetty
+ *
+
+
+ javax.servlet.jsp
+ *
+
+
+ javax.servlet
+ *
+
+
+
+
+
+ org.apache.hadoop
+ hadoop-hdfs
+ tests
+
+
+ javax.servlet
+ *
+
+
+ netty
+ io.netty
+
+
+ netty-all
+ io.netty
+
+
+
+
+
+
+ ${hive.groupid}
+ hive-exec
+ ${hive.version}
+ ${hive.exec.classifier}
+
+
+ javax.servlet
+ servlet-api
+
+
+
+
+
+ ${hive.groupid}
+ hive-jdbc
+ ${hive.version}
+
+
+ org.slf4j
+ slf4j-api
+
+
+ javax.servlet.jsp
+ *
+
+
+ javax.servlet
+ *
+
+
+ org.eclipse.jetty
+ *
+
+
+ test
+
+
org.awaitility
awaitility
@@ -138,6 +337,12 @@
test
+
+ org.mockito
+ mockito-junit-jupiter
+ test
+
+
diff --git a/hudi-integ-test/prepare_integration_suite.sh b/hudi-integ-test/prepare_integration_suite.sh
new file mode 100644
index 000000000..06999d9ca
--- /dev/null
+++ b/hudi-integ-test/prepare_integration_suite.sh
@@ -0,0 +1,174 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#!/bin/bash
+
+# Determine the current working directory
+_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+# Preserve the calling directory
+_CALLING_DIR="$(pwd)"
+
+#########################
+# The command line help #
+#########################
+usage() {
+ echo "Usage: $0"
+ echo " --spark-command, prints the spark command"
+ echo " -h | --hadoop, hadoop-version"
+ echo " -s | --spark, spark version"
+ echo " -p | --parquet, parquet version"
+ echo " -a | --avro, avro version"
+ echo " -i | --hive, hive version"
+ echo " -l | --scala, scala version"
+ exit 1
+}
+
+get_spark_command() {
+if [ -z "$scala" ]
+then
+ scala="2.11"
+else
+ scala=$1
+fi
+echo "spark-submit --packages org.apache.spark:spark-avro_${scala}:2.4.4 \
+--master $0 \
+--deploy-mode $1 \
+--properties-file $2 \
+--class org.apache.hudi.integ.testsuite.HoodieTestSuiteJob \
+`ls target/hudi-integ-test-*-SNAPSHOT.jar` \
+--source-class $3 \
+--source-ordering-field $4 \
+--input-base-path $5 \
+--target-base-path $6 \
+--target-table $7 \
+--props $8 \
+--storage-type $9 \
+--payload-class "${10}" \
+--workload-yaml-path "${11}" \
+--input-file-size "${12}" \
+--"
+}
+
+case "$1" in
+ --help)
+ usage
+ exit 0
+ ;;
+esac
+
+case "$1" in
+ --spark-command)
+ get_spark_command
+ exit 0
+ ;;
+esac
+
+while getopts ":h:s:p:a:i:l:-:" opt; do
+ case $opt in
+ h) hadoop="$OPTARG"
+ printf "Argument hadoop is %s\n" "$hadoop"
+ ;;
+ s) spark="$OPTARG"
+ printf "Argument spark is %s\n" "$spark"
+ ;;
+ p) parquet="$OPTARG"
+ printf "Argument parquet is %s\n" "$parquet"
+ ;;
+ a) avro="$OPTARG"
+ printf "Argument avro is %s\n" "$avro"
+ ;;
+ i) hive="$OPTARG"
+ printf "Argument hive is %s\n" "$hive"
+ ;;
+ l) scala="$OPTARG"
+ printf "Argument scala is %s\n" "$scala"
+ ;;
+ -)
+ case "$OPTARG" in
+ hadoop)
+ hadoop="${!OPTIND}"; OPTIND=$(( $OPTIND + 1 ))
+ printf "Argument hadoop is %s\n" "$hadoop"
+ ;;
+ spark)
+ spark="${!OPTIND}"; OPTIND=$(( $OPTIND + 1 ))
+ printf "Argument spark is %s\n" "$spark"
+ ;;
+ parquet)
+ parquet="${!OPTIND}"; OPTIND=$(( $OPTIND + 1 ))
+ printf "Argument parquet is %s\n" "$parquet"
+ ;;
+ avro)
+ avro="${!OPTIND}"; OPTIND=$(( $OPTIND + 1 ))
+ printf "Argument avro is %s\n" "$avro"
+ ;;
+ hive)
+ hive="${!OPTIND}"; OPTIND=$(( $OPTIND + 1 ))
+ printf "Argument hive is %s\n" "$hive"
+ ;;
+ scala)
+ scala="${!OPTIND}"; OPTIND=$(( $OPTIND + 1 ))
+ printf "Argument scala is %s\n" "$scala"
+ ;;
+ *) echo "Invalid option --$OPTARG" >&2
+ ;;
+ esac ;;
+ \?) echo "Invalid option -$OPTARG" >&2
+ ;;
+ esac
+done
+
+
+get_versions () {
+ base_command=''
+ if [ -z "$hadoop" ]
+ then
+ base_command=$base_command
+ else
+ hadoop=$1
+ base_command+=' -Dhadoop.version='$hadoop
+ fi
+
+ if [ -z "$hive" ]
+ then
+ base_command=$base_command
+ else
+ hive=$2
+ base_command+=' -Dhive.version='$hive
+ fi
+
+ if [ -z "$scala" ]
+ then
+ base_command=$base_command
+ else
+ scala=$3
+ base_command+=' -Dscala-'$scala
+ fi
+ echo $base_command
+}
+
+versions=$(get_versions $hadoop $hive $scala)
+
+final_command='mvn clean install -DskipTests '$versions
+printf "Final command $final_command \n"
+
+# change to the project root directory to run maven command
+move_to_root='cd ..'
+$move_to_root && $final_command
+
+# change back to original working directory
+cd $_CALLING_DIR
+
+printf "A sample spark command to start the integration suite \n"
+get_spark_command $scala
\ No newline at end of file
diff --git a/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/HoodieDeltaStreamerWrapper.java b/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/HoodieDeltaStreamerWrapper.java
new file mode 100644
index 000000000..a489bacc3
--- /dev/null
+++ b/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/HoodieDeltaStreamerWrapper.java
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.integ.testsuite;
+
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hudi.client.WriteStatus;
+import org.apache.hudi.common.model.HoodieRecord;
+import org.apache.hudi.common.util.collection.Pair;
+import org.apache.hudi.utilities.deltastreamer.HoodieDeltaStreamer;
+import org.apache.hudi.utilities.schema.SchemaProvider;
+import org.apache.spark.api.java.JavaRDD;
+import org.apache.spark.api.java.JavaSparkContext;
+
+/**
+ * Extends the {@link HoodieDeltaStreamer} to expose certain operations helpful in running the Test Suite.
+ * This is done to achieve 2 things 1) Leverage some components of {@link HoodieDeltaStreamer} 2)
+ * Piggyback on the suite to test {@link HoodieDeltaStreamer}
+ */
+public class HoodieDeltaStreamerWrapper extends HoodieDeltaStreamer {
+
+ public HoodieDeltaStreamerWrapper(Config cfg, JavaSparkContext jssc) throws Exception {
+ super(cfg, jssc);
+ }
+
+ public HoodieDeltaStreamerWrapper(Config cfg, JavaSparkContext jssc, FileSystem fs, HiveConf conf) throws Exception {
+ super(cfg, jssc, fs, conf);
+ }
+
+ public JavaRDD upsert(Operation operation) throws
+ Exception {
+ cfg.operation = operation;
+ return deltaSyncService.getDeltaSync().syncOnce().getRight();
+ }
+
+ public JavaRDD insert() throws Exception {
+ return upsert(Operation.INSERT);
+ }
+
+ public JavaRDD bulkInsert() throws
+ Exception {
+ return upsert(Operation.BULK_INSERT);
+ }
+
+ public void scheduleCompact() throws Exception {
+ // Since we don't support scheduleCompact() operation in delta-streamer, assume upsert without any data that will
+ // trigger scheduling compaction
+ upsert(Operation.UPSERT);
+ }
+
+ public JavaRDD compact() throws Exception {
+ // Since we don't support compact() operation in delta-streamer, assume upsert without any data that will trigger
+ // inline compaction
+ return upsert(Operation.UPSERT);
+ }
+
+ public Pair>> fetchSource() throws Exception {
+ return deltaSyncService.getDeltaSync().readFromSource(deltaSyncService.getDeltaSync().getCommitTimelineOpt());
+ }
+
+}
diff --git a/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/HoodieTestSuiteJob.java b/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/HoodieTestSuiteJob.java
new file mode 100644
index 000000000..be036c16d
--- /dev/null
+++ b/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/HoodieTestSuiteJob.java
@@ -0,0 +1,186 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.integ.testsuite;
+
+import com.beust.jcommander.JCommander;
+import com.beust.jcommander.Parameter;
+import java.io.IOException;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hudi.DataSourceUtils;
+import org.apache.hudi.common.config.SerializableConfiguration;
+import org.apache.hudi.common.config.TypedProperties;
+import org.apache.hudi.common.fs.FSUtils;
+import org.apache.hudi.common.model.HoodieTableType;
+import org.apache.hudi.common.table.HoodieTableMetaClient;
+import org.apache.hudi.common.util.ReflectionUtils;
+import org.apache.hudi.exception.HoodieException;
+import org.apache.hudi.integ.testsuite.configuration.DFSDeltaConfig;
+import org.apache.hudi.integ.testsuite.dag.DagUtils;
+import org.apache.hudi.integ.testsuite.dag.WorkflowDag;
+import org.apache.hudi.integ.testsuite.dag.WorkflowDagGenerator;
+import org.apache.hudi.integ.testsuite.dag.scheduler.DagScheduler;
+import org.apache.hudi.integ.testsuite.generator.DeltaGenerator;
+import org.apache.hudi.integ.testsuite.reader.DeltaInputType;
+import org.apache.hudi.integ.testsuite.writer.DeltaOutputMode;
+import org.apache.hudi.keygen.KeyGenerator;
+import org.apache.hudi.utilities.UtilHelpers;
+import org.apache.hudi.utilities.deltastreamer.HoodieDeltaStreamer;
+import org.apache.hudi.utilities.schema.SchemaProvider;
+import org.apache.spark.api.java.JavaSparkContext;
+import org.apache.spark.sql.SparkSession;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * This is the entry point for running a Hudi Test Suite. Although this class has similarities with
+ * {@link HoodieDeltaStreamer} this class does not extend it since do not want to create a dependency on the changes in
+ * DeltaStreamer.
+ */
+public class HoodieTestSuiteJob {
+
+ private static volatile Logger log = LoggerFactory.getLogger(HoodieTestSuiteJob.class);
+
+ private final HoodieTestSuiteConfig cfg;
+ /**
+ * Bag of properties with source, hoodie client, key generator etc.
+ */
+ TypedProperties props;
+ /**
+ * Schema provider that supplies the command for writing out the generated payloads.
+ */
+ private transient SchemaProvider schemaProvider;
+ /**
+ * Filesystem used.
+ */
+ private transient FileSystem fs;
+ /**
+ * Spark context.
+ */
+ private transient JavaSparkContext jsc;
+ /**
+ * Spark Session.
+ */
+ private transient SparkSession sparkSession;
+ /**
+ * Hive Config.
+ */
+ private transient HiveConf hiveConf;
+
+ private KeyGenerator keyGenerator;
+
+ public HoodieTestSuiteJob(HoodieTestSuiteConfig cfg, JavaSparkContext jsc) throws IOException {
+ this.cfg = cfg;
+ this.jsc = jsc;
+ this.sparkSession = SparkSession.builder().config(jsc.getConf()).getOrCreate();
+ this.fs = FSUtils.getFs(cfg.inputBasePath, jsc.hadoopConfiguration());
+ this.props = UtilHelpers.readConfig(fs, new Path(cfg.propsFilePath), cfg.configs).getConfig();
+ log.info("Creating workload generator with configs : {}", props.toString());
+ this.schemaProvider = UtilHelpers.createSchemaProvider(cfg.schemaProviderClassName, props, jsc);
+ this.hiveConf = getDefaultHiveConf(jsc.hadoopConfiguration());
+ this.keyGenerator = DataSourceUtils.createKeyGenerator(props);
+ if (!fs.exists(new Path(cfg.targetBasePath))) {
+ HoodieTableMetaClient.initTableType(jsc.hadoopConfiguration(), cfg.targetBasePath,
+ HoodieTableType.valueOf(cfg.tableType), cfg.targetTableName, "archived");
+ }
+ }
+
+ private static HiveConf getDefaultHiveConf(Configuration cfg) {
+ HiveConf hiveConf = new HiveConf();
+ hiveConf.addResource(cfg);
+ return hiveConf;
+ }
+
+ public static void main(String[] args) throws Exception {
+ final HoodieTestSuiteConfig cfg = new HoodieTestSuiteConfig();
+ JCommander cmd = new JCommander(cfg, args);
+ if (cfg.help || args.length == 0) {
+ cmd.usage();
+ System.exit(1);
+ }
+
+ JavaSparkContext jssc = UtilHelpers.buildSparkContext("workload-generator-" + cfg.outputTypeName
+ + "-" + cfg.inputFormatName, cfg.sparkMaster);
+ new HoodieTestSuiteJob(cfg, jssc).runTestSuite();
+ }
+
+ public void runTestSuite() {
+ try {
+ WorkflowDag workflowDag = this.cfg.workloadYamlPath == null ? ((WorkflowDagGenerator) ReflectionUtils
+ .loadClass((this.cfg).workloadDagGenerator)).build()
+ : DagUtils.convertYamlPathToDag(this.fs, this.cfg.workloadYamlPath);
+ log.info("Workflow Dag => " + DagUtils.convertDagToYaml(workflowDag));
+ long startTime = System.currentTimeMillis();
+ String schemaStr = schemaProvider.getSourceSchema().toString();
+ final HoodieTestSuiteWriter writer = new HoodieTestSuiteWriter(jsc, props, cfg, schemaStr);
+ final DeltaGenerator deltaGenerator = new DeltaGenerator(
+ new DFSDeltaConfig(DeltaOutputMode.valueOf(cfg.outputTypeName), DeltaInputType.valueOf(cfg.inputFormatName),
+ new SerializableConfiguration(jsc.hadoopConfiguration()), cfg.inputBasePath, cfg.targetBasePath,
+ schemaStr, cfg.limitFileSize), jsc, sparkSession, schemaStr, keyGenerator);
+ DagScheduler dagScheduler = new DagScheduler(workflowDag, writer, deltaGenerator);
+ dagScheduler.schedule();
+ log.info("Finished scheduling all tasks, Time taken {}", System.currentTimeMillis() - startTime);
+ } catch (Exception e) {
+ log.error("Failed to run Test Suite ", e);
+ throw new HoodieException("Failed to run Test Suite ", e);
+ } finally {
+ jsc.stop();
+ }
+ }
+
+ /**
+ * The Hudi test suite uses {@link HoodieDeltaStreamer} to run some operations hence extend delta streamer config.
+ */
+ public static class HoodieTestSuiteConfig extends HoodieDeltaStreamer.Config {
+
+ @Parameter(names = {"--input-base-path"}, description = "base path for input data"
+ + "(Will be created if did not exist first time around. If exists, more data will be added to that path)",
+ required = true)
+ public String inputBasePath;
+
+ @Parameter(names = {
+ "--workload-generator-classname"}, description = "WorkflowDag of operations to generate the workload",
+ required = true)
+ public String workloadDagGenerator = WorkflowDagGenerator.class.getName();
+
+ @Parameter(names = {
+ "--workload-yaml-path"}, description = "Workflow Dag yaml path to generate the workload")
+ public String workloadYamlPath;
+
+ @Parameter(names = {"--delta-output-type"}, description = "Subclass of "
+ + "org.apache.hudi.testsuite.workload.DeltaOutputMode to readAvro data.")
+ public String outputTypeName = DeltaOutputMode.DFS.name();
+
+ @Parameter(names = {"--delta-input-format"}, description = "Subclass of "
+ + "org.apache.hudi.testsuite.workload.DeltaOutputMode to read avro data.")
+ public String inputFormatName = DeltaInputType.AVRO.name();
+
+ @Parameter(names = {"--input-file-size"}, description = "The min/max size of the input files to generate",
+ required = true)
+ public Long limitFileSize = 1024 * 1024 * 120L;
+
+ @Parameter(names = {"--use-deltastreamer"}, description = "Choose whether to use HoodieDeltaStreamer to "
+ + "perform"
+ + " ingestion. If set to false, HoodieWriteClient will be used")
+ public Boolean useDeltaStreamer = false;
+
+ }
+}
diff --git a/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/HoodieTestSuiteWriter.java b/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/HoodieTestSuiteWriter.java
new file mode 100644
index 000000000..b22faca22
--- /dev/null
+++ b/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/HoodieTestSuiteWriter.java
@@ -0,0 +1,219 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.integ.testsuite;
+
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Properties;
+import java.util.Set;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hudi.avro.model.HoodieCompactionPlan;
+import org.apache.hudi.client.HoodieReadClient;
+import org.apache.hudi.client.HoodieWriteClient;
+import org.apache.hudi.client.WriteStatus;
+import org.apache.hudi.common.model.HoodieRecord;
+import org.apache.hudi.common.table.timeline.HoodieActiveTimeline;
+import org.apache.hudi.common.util.Option;
+import org.apache.hudi.common.util.collection.Pair;
+import org.apache.hudi.config.HoodieCompactionConfig;
+import org.apache.hudi.config.HoodieIndexConfig;
+import org.apache.hudi.config.HoodieWriteConfig;
+import org.apache.hudi.index.HoodieIndex;
+import org.apache.hudi.integ.testsuite.dag.nodes.CleanNode;
+import org.apache.hudi.integ.testsuite.dag.nodes.DagNode;
+import org.apache.hudi.integ.testsuite.dag.nodes.RollbackNode;
+import org.apache.hudi.integ.testsuite.dag.nodes.ScheduleCompactNode;
+import org.apache.hudi.integ.testsuite.HoodieTestSuiteJob.HoodieTestSuiteConfig;
+import org.apache.hudi.utilities.deltastreamer.HoodieDeltaStreamer.Operation;
+import org.apache.hudi.utilities.schema.SchemaProvider;
+import org.apache.spark.api.java.JavaRDD;
+import org.apache.spark.api.java.JavaSparkContext;
+
+/**
+ * A writer abstraction for the Hudi test suite. This class wraps different implementations of writers used to perform
+ * write operations into the target hudi dataset. Current supported writers are {@link HoodieDeltaStreamerWrapper}
+ * and {@link HoodieWriteClient}.
+ */
+public class HoodieTestSuiteWriter {
+
+ private HoodieDeltaStreamerWrapper deltaStreamerWrapper;
+ private HoodieWriteClient writeClient;
+ protected HoodieTestSuiteConfig cfg;
+ private Option lastCheckpoint;
+ private HoodieReadClient hoodieReadClient;
+ private Properties props;
+ private String schema;
+ private transient Configuration configuration;
+ private transient JavaSparkContext sparkContext;
+ private static Set VALID_DAG_NODES_TO_ALLOW_WRITE_CLIENT_IN_DELTASTREAMER_MODE = new HashSet<>(
+ Arrays.asList(RollbackNode.class.getName(), CleanNode.class.getName(), ScheduleCompactNode.class.getName()));
+
+ public HoodieTestSuiteWriter(JavaSparkContext jsc, Properties props, HoodieTestSuiteConfig cfg, String schema) throws
+ Exception {
+ this(jsc, props, cfg, schema, true);
+ }
+
+ public HoodieTestSuiteWriter(JavaSparkContext jsc, Properties props, HoodieTestSuiteConfig cfg, String schema,
+ boolean rollbackInflight) throws Exception {
+ // We ensure that only 1 instance of HoodieWriteClient is instantiated for a HoodieTestSuiteWriter
+ // This does not instantiate a HoodieWriteClient until a
+ // {@link HoodieDeltaStreamer#commit(HoodieWriteClient, JavaRDD, Option)} is invoked.
+ this.deltaStreamerWrapper = new HoodieDeltaStreamerWrapper(cfg, jsc);
+ this.hoodieReadClient = new HoodieReadClient(jsc, cfg.targetBasePath);
+ if (!cfg.useDeltaStreamer) {
+ this.writeClient = new HoodieWriteClient(jsc, getHoodieClientConfig(cfg, props, schema), rollbackInflight);
+ }
+ this.cfg = cfg;
+ this.configuration = jsc.hadoopConfiguration();
+ this.sparkContext = jsc;
+ this.props = props;
+ this.schema = schema;
+ }
+
+ private HoodieWriteConfig getHoodieClientConfig(HoodieTestSuiteConfig cfg, Properties props, String schema) {
+ HoodieWriteConfig.Builder builder =
+ HoodieWriteConfig.newBuilder().combineInput(true, true).withPath(cfg.targetBasePath)
+ .withAutoCommit(false)
+ .withCompactionConfig(HoodieCompactionConfig.newBuilder().withPayloadClass(cfg.payloadClassName).build())
+ .forTable(cfg.targetTableName)
+ .withIndexConfig(HoodieIndexConfig.newBuilder().withIndexType(HoodieIndex.IndexType.BLOOM).build())
+ .withProps(props);
+ builder = builder.withSchema(schema);
+ return builder.build();
+ }
+
+ private boolean allowWriteClientAccess(DagNode dagNode) {
+ if (VALID_DAG_NODES_TO_ALLOW_WRITE_CLIENT_IN_DELTASTREAMER_MODE.contains(dagNode.getClass().getName())) {
+ return true;
+ }
+ return false;
+ }
+
+ public Pair>> fetchSource() throws Exception {
+ return this.deltaStreamerWrapper.fetchSource();
+ }
+
+ public Option startCommit() {
+ if (cfg.useDeltaStreamer) {
+ return Option.of(HoodieActiveTimeline.createNewInstantTime());
+ } else {
+ return Option.of(writeClient.startCommit());
+ }
+ }
+
+ public JavaRDD upsert(Option instantTime) throws Exception {
+ if (cfg.useDeltaStreamer) {
+ return deltaStreamerWrapper.upsert(Operation.UPSERT);
+ } else {
+ Pair>> nextBatch = fetchSource();
+ lastCheckpoint = Option.of(nextBatch.getValue().getLeft());
+ return writeClient.upsert(nextBatch.getRight().getRight(), instantTime.get());
+ }
+ }
+
+ public JavaRDD insert(Option instantTime) throws Exception {
+ if (cfg.useDeltaStreamer) {
+ return deltaStreamerWrapper.insert();
+ } else {
+ Pair>> nextBatch = fetchSource();
+ lastCheckpoint = Option.of(nextBatch.getValue().getLeft());
+ return writeClient.insert(nextBatch.getRight().getRight(), instantTime.get());
+ }
+ }
+
+ public JavaRDD bulkInsert(Option instantTime) throws Exception {
+ if (cfg.useDeltaStreamer) {
+ return deltaStreamerWrapper.bulkInsert();
+ } else {
+ Pair>> nextBatch = fetchSource();
+ lastCheckpoint = Option.of(nextBatch.getValue().getLeft());
+ return writeClient.bulkInsert(nextBatch.getRight().getRight(), instantTime.get());
+ }
+ }
+
+ public JavaRDD compact(Option instantTime) throws Exception {
+ if (cfg.useDeltaStreamer) {
+ return deltaStreamerWrapper.compact();
+ } else {
+ if (!instantTime.isPresent()) {
+ Option> compactionPlanPair = Option
+ .fromJavaOptional(hoodieReadClient.getPendingCompactions()
+ .stream().findFirst());
+ if (compactionPlanPair.isPresent()) {
+ instantTime = Option.of(compactionPlanPair.get().getLeft());
+ }
+ }
+ if (instantTime.isPresent()) {
+ return writeClient.compact(instantTime.get());
+ } else {
+ return null;
+ }
+ }
+ }
+
+ public Option scheduleCompaction(Option> previousCommitExtraMetadata) throws
+ Exception {
+ if (!cfg.useDeltaStreamer) {
+ deltaStreamerWrapper.scheduleCompact();
+ return Option.empty();
+ } else {
+ return writeClient.scheduleCompaction(previousCommitExtraMetadata);
+ }
+ }
+
+ public void commit(JavaRDD records, Option instantTime) {
+ if (!cfg.useDeltaStreamer) {
+ Map extraMetadata = new HashMap<>();
+ /** Store the checkpoint in the commit metadata just like
+ * {@link HoodieDeltaStreamer#commit(HoodieWriteClient, JavaRDD, Option)} **/
+ extraMetadata.put(HoodieDeltaStreamerWrapper.CHECKPOINT_KEY, lastCheckpoint.get());
+ writeClient.commit(instantTime.get(), records, Option.of(extraMetadata));
+ }
+ }
+
+ public HoodieWriteClient getWriteClient(DagNode dagNode) throws IllegalAccessException {
+ if (cfg.useDeltaStreamer & !allowWriteClientAccess(dagNode)) {
+ throw new IllegalAccessException("cannot access write client when testing in deltastreamer mode");
+ }
+ synchronized (this) {
+ if (writeClient == null) {
+ this.writeClient = new HoodieWriteClient(this.sparkContext, getHoodieClientConfig(cfg, props, schema), false);
+ }
+ }
+ return writeClient;
+ }
+
+ public HoodieDeltaStreamerWrapper getDeltaStreamerWrapper() {
+ return deltaStreamerWrapper;
+ }
+
+ public HoodieTestSuiteConfig getCfg() {
+ return cfg;
+ }
+
+ public Configuration getConfiguration() {
+ return configuration;
+ }
+
+ public JavaSparkContext getSparkContext() {
+ return sparkContext;
+ }
+}
diff --git a/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/configuration/DFSDeltaConfig.java b/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/configuration/DFSDeltaConfig.java
new file mode 100644
index 000000000..291562808
--- /dev/null
+++ b/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/configuration/DFSDeltaConfig.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.integ.testsuite.configuration;
+
+import org.apache.hudi.common.config.SerializableConfiguration;
+import org.apache.hudi.integ.testsuite.reader.DeltaInputType;
+import org.apache.hudi.integ.testsuite.writer.DeltaOutputMode;
+
+/**
+ * Configuration to hold details about a DFS based output type, implements {@link DeltaConfig}.
+ */
+public class DFSDeltaConfig extends DeltaConfig {
+
+ // The base path where the generated data should be written to. This data will in turn be used to write into a hudi
+ // dataset
+ private final String deltaBasePath;
+ private final String datasetOutputPath;
+ private final String schemaStr;
+ // Maximum file size for the files generated
+ private final Long maxFileSize;
+ // The current batch id
+ private Integer batchId;
+
+ public DFSDeltaConfig(DeltaOutputMode deltaOutputMode, DeltaInputType deltaInputType,
+ SerializableConfiguration configuration,
+ String deltaBasePath, String targetBasePath, String schemaStr, Long maxFileSize) {
+ super(deltaOutputMode, deltaInputType, configuration);
+ this.deltaBasePath = deltaBasePath;
+ this.schemaStr = schemaStr;
+ this.maxFileSize = maxFileSize;
+ this.datasetOutputPath = targetBasePath;
+ }
+
+ public String getDeltaBasePath() {
+ return deltaBasePath;
+ }
+
+ public String getDatasetOutputPath() {
+ return datasetOutputPath;
+ }
+
+ public String getSchemaStr() {
+ return schemaStr;
+ }
+
+ public Long getMaxFileSize() {
+ return maxFileSize;
+ }
+
+ public Integer getBatchId() {
+ return batchId;
+ }
+
+ public void setBatchId(Integer batchId) {
+ this.batchId = batchId;
+ }
+}
diff --git a/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/configuration/DeltaConfig.java b/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/configuration/DeltaConfig.java
new file mode 100644
index 000000000..30fa58442
--- /dev/null
+++ b/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/configuration/DeltaConfig.java
@@ -0,0 +1,256 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.integ.testsuite.configuration;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hudi.common.config.SerializableConfiguration;
+import org.apache.hudi.common.util.collection.Pair;
+import org.apache.hudi.integ.testsuite.reader.DeltaInputType;
+import org.apache.hudi.integ.testsuite.writer.DeltaOutputMode;
+
+/**
+ * Configuration to hold the delta output type and delta input format.
+ */
+public class DeltaConfig implements Serializable {
+
+ private final DeltaOutputMode deltaOutputMode;
+ private final DeltaInputType deltaInputType;
+ private final SerializableConfiguration configuration;
+
+ public DeltaConfig(DeltaOutputMode deltaOutputMode, DeltaInputType deltaInputType,
+ SerializableConfiguration configuration) {
+ this.deltaOutputMode = deltaOutputMode;
+ this.deltaInputType = deltaInputType;
+ this.configuration = configuration;
+ }
+
+ public DeltaOutputMode getDeltaOutputMode() {
+ return deltaOutputMode;
+ }
+
+ public DeltaInputType getDeltaInputType() {
+ return deltaInputType;
+ }
+
+ public Configuration getConfiguration() {
+ return configuration.get();
+ }
+
+ /**
+ * Represents any kind of workload operation for new data. Each workload also contains a set of Option sequence of
+ * actions that can be executed in parallel.
+ */
+ public static class Config {
+
+ public static final String CONFIG_NAME = "config";
+ public static final String TYPE = "type";
+ public static final String NODE_NAME = "name";
+ public static final String DEPENDENCIES = "deps";
+ public static final String CHILDREN = "children";
+ public static final String HIVE_QUERIES = "hive_queries";
+ public static final String HIVE_PROPERTIES = "hive_props";
+ private static String NUM_RECORDS_INSERT = "num_records_insert";
+ private static String NUM_RECORDS_UPSERT = "num_records_upsert";
+ private static String REPEAT_COUNT = "repeat_count";
+ private static String RECORD_SIZE = "record_size";
+ private static String NUM_PARTITIONS_INSERT = "num_partitions_insert";
+ private static String NUM_PARTITIONS_UPSERT = "num_partitions_upsert";
+ private static String NUM_FILES_UPSERT = "num_files_upsert";
+ private static String FRACTION_UPSERT_PER_FILE = "fraction_upsert_per_file";
+ private static String DISABLE_GENERATE = "disable_generate";
+ private static String DISABLE_INGEST = "disable_ingest";
+ private static String HIVE_LOCAL = "hive_local";
+
+ private Map configsMap;
+
+ public Config(Map configsMap) {
+ this.configsMap = configsMap;
+ }
+
+ public static Builder newBuilder() {
+ return new Builder();
+ }
+
+ public long getNumRecordsInsert() {
+ return Long.valueOf(configsMap.getOrDefault(NUM_RECORDS_INSERT, 0).toString());
+ }
+
+ public long getNumRecordsUpsert() {
+ return Long.valueOf(configsMap.getOrDefault(NUM_RECORDS_UPSERT, 0).toString());
+ }
+
+ public int getRecordSize() {
+ return Integer.valueOf(configsMap.getOrDefault(RECORD_SIZE, 1024).toString());
+ }
+
+ public int getNumInsertPartitions() {
+ return Integer.valueOf(configsMap.getOrDefault(NUM_PARTITIONS_INSERT, 1).toString());
+ }
+
+ public int getRepeatCount() {
+ return Integer.valueOf(configsMap.getOrDefault(REPEAT_COUNT, 1).toString());
+ }
+
+ public int getNumUpsertPartitions() {
+ return Integer.valueOf(configsMap.getOrDefault(NUM_PARTITIONS_UPSERT, 0).toString());
+ }
+
+ public int getNumUpsertFiles() {
+ return Integer.valueOf(configsMap.getOrDefault(NUM_FILES_UPSERT, 1).toString());
+ }
+
+ public double getFractionUpsertPerFile() {
+ return Double.valueOf(configsMap.getOrDefault(FRACTION_UPSERT_PER_FILE, 0.0).toString());
+ }
+
+ public boolean isDisableGenerate() {
+ return Boolean.valueOf(configsMap.getOrDefault(DISABLE_GENERATE, false).toString());
+ }
+
+ public boolean isDisableIngest() {
+ return Boolean.valueOf(configsMap.getOrDefault(DISABLE_INGEST, false).toString());
+ }
+
+ public Map getOtherConfigs() {
+ if (configsMap == null) {
+ return new HashMap<>();
+ }
+ return configsMap;
+ }
+
+ public List> getHiveQueries() {
+ try {
+ return (List>) this.configsMap.getOrDefault("hive_queries", new ArrayList<>());
+ } catch (Exception e) {
+ throw new RuntimeException("unable to get hive queries from configs");
+ }
+ }
+
+ public boolean isHiveLocal() {
+ return Boolean.valueOf(configsMap.getOrDefault(HIVE_LOCAL, false).toString());
+ }
+
+ public List getHiveProperties() {
+ return (List) this.configsMap.getOrDefault(HIVE_PROPERTIES, new ArrayList<>());
+ }
+
+ @Override
+ public String toString() {
+ try {
+ return new ObjectMapper().writeValueAsString(this.configsMap);
+ } catch (Exception e) {
+ throw new RuntimeException("unable to generate string representation of config");
+ }
+ }
+
+ public static class Builder {
+
+ private Map configsMap = new HashMap<>();
+
+ public Builder() {
+ }
+
+ public Builder withNumRecordsToInsert(long numRecordsInsert) {
+ this.configsMap.put(NUM_RECORDS_INSERT, numRecordsInsert);
+ return this;
+ }
+
+ public Builder withNumRecordsToUpdate(long numRecordsUpsert) {
+ this.configsMap.put(NUM_RECORDS_UPSERT, numRecordsUpsert);
+ return this;
+ }
+
+ public Builder withNumInsertPartitions(int numInsertPartitions) {
+ this.configsMap.put(NUM_PARTITIONS_INSERT, numInsertPartitions);
+ return this;
+ }
+
+ public Builder withNumUpsertPartitions(int numUpsertPartitions) {
+ this.configsMap.put(NUM_PARTITIONS_UPSERT, numUpsertPartitions);
+ return this;
+ }
+
+ public Builder withNumUpsertFiles(int numUpsertFiles) {
+ this.configsMap.put(NUM_FILES_UPSERT, numUpsertFiles);
+ return this;
+ }
+
+ public Builder withFractionUpsertPerFile(double fractionUpsertPerFile) {
+ this.configsMap.put(FRACTION_UPSERT_PER_FILE, fractionUpsertPerFile);
+ return this;
+ }
+
+ public Builder withNumTimesToRepeat(int repeatCount) {
+ this.configsMap.put(REPEAT_COUNT, repeatCount);
+ return this;
+ }
+
+ public Builder withRecordSize(int recordSize) {
+ this.configsMap.put(RECORD_SIZE, recordSize);
+ return this;
+ }
+
+ public Builder disableGenerate(boolean generate) {
+ this.configsMap.put(DISABLE_GENERATE, generate);
+ return this;
+ }
+
+ public Builder disableIngest(boolean ingest) {
+ this.configsMap.put(DISABLE_INGEST, ingest);
+ return this;
+ }
+
+ public Builder withConfig(String name, Object value) {
+ this.configsMap.put(name, value);
+ return this;
+ }
+
+ public Builder withHiveQueryAndResults(List> hiveQueries) {
+ this.configsMap.put(HIVE_QUERIES, hiveQueries);
+ return this;
+ }
+
+ public Builder withHiveLocal(boolean startHiveLocal) {
+ this.configsMap.put(HIVE_LOCAL, startHiveLocal);
+ return this;
+ }
+
+ public Builder withHiveProperties(List hiveProperties) {
+ this.configsMap.put(HIVE_PROPERTIES, hiveProperties);
+ return this;
+ }
+
+ public Builder withConfigsMap(Map configsMap) {
+ this.configsMap = configsMap;
+ return this;
+ }
+
+ public Config build() {
+ return new Config(configsMap);
+ }
+
+ }
+ }
+}
diff --git a/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/converter/Converter.java b/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/converter/Converter.java
new file mode 100644
index 000000000..e4ad0a7fa
--- /dev/null
+++ b/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/converter/Converter.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.integ.testsuite.converter;
+
+import java.io.Serializable;
+import org.apache.spark.api.java.JavaRDD;
+
+/**
+ * Implementations of {@link Converter} will convert data from one format to another.
+ *
+ * @param Input Data Type
+ * @param Output Data Type
+ */
+public interface Converter extends Serializable {
+
+ JavaRDD convert(JavaRDD inputRDD);
+}
\ No newline at end of file
diff --git a/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/converter/UpdateConverter.java b/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/converter/UpdateConverter.java
new file mode 100644
index 000000000..24520a362
--- /dev/null
+++ b/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/converter/UpdateConverter.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.integ.testsuite.converter;
+
+import java.util.List;
+import org.apache.avro.generic.GenericRecord;
+import org.apache.hudi.integ.testsuite.generator.LazyRecordGeneratorIterator;
+import org.apache.hudi.integ.testsuite.generator.UpdateGeneratorIterator;
+import org.apache.spark.api.java.JavaRDD;
+
+/**
+ * This converter creates an update {@link GenericRecord} from an existing {@link GenericRecord}.
+ */
+public class UpdateConverter implements Converter {
+
+ private final String schemaStr;
+ // The fields that should not be mutated when converting the insert record to an update record, generally the
+ // record_key
+ private final List partitionPathFields;
+ private final List recordKeyFields;
+ private final int minPayloadSize;
+
+ public UpdateConverter(String schemaStr, int minPayloadSize, List partitionPathFields,
+ List recordKeyFields) {
+ this.schemaStr = schemaStr;
+ this.partitionPathFields = partitionPathFields;
+ this.recordKeyFields = recordKeyFields;
+ this.minPayloadSize = minPayloadSize;
+ }
+
+ @Override
+ public JavaRDD convert(JavaRDD inputRDD) {
+ return inputRDD.mapPartitions(recordItr -> new LazyRecordGeneratorIterator(new UpdateGeneratorIterator(recordItr,
+ schemaStr, partitionPathFields, recordKeyFields, minPayloadSize)));
+ }
+
+}
diff --git a/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/dag/DagUtils.java b/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/dag/DagUtils.java
new file mode 100644
index 000000000..288986719
--- /dev/null
+++ b/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/dag/DagUtils.java
@@ -0,0 +1,217 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.integ.testsuite.dag;
+
+import com.fasterxml.jackson.core.JsonParser;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.fasterxml.jackson.dataformat.yaml.YAMLFactory;
+import com.fasterxml.jackson.dataformat.yaml.YAMLGenerator.Feature;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.stream.Collectors;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hudi.common.util.ReflectionUtils;
+import org.apache.hudi.common.util.StringUtils;
+import org.apache.hudi.common.util.collection.Pair;
+import org.apache.hudi.integ.testsuite.configuration.DeltaConfig;
+import org.apache.hudi.integ.testsuite.dag.nodes.DagNode;
+
+/**
+ * Utility class to SerDe workflow dag.
+ */
+public class DagUtils {
+
+ static final ObjectMapper MAPPER = new ObjectMapper();
+
+ /**
+ * Converts a YAML path to {@link WorkflowDag}.
+ */
+ public static WorkflowDag convertYamlPathToDag(FileSystem fs, String path) throws IOException {
+ InputStream is = fs.open(new Path(path));
+ return convertYamlToDag(toString(is));
+ }
+
+ /**
+ * Converts a YAML representation to {@link WorkflowDag}.
+ */
+ public static WorkflowDag convertYamlToDag(String yaml) throws IOException {
+ Map allNodes = new HashMap<>();
+ final ObjectMapper yamlReader = new ObjectMapper(new YAMLFactory());
+ final JsonNode jsonNode = yamlReader.readTree(yaml);
+ Iterator> itr = jsonNode.fields();
+ while (itr.hasNext()) {
+ Entry dagNode = itr.next();
+ allNodes.put(dagNode.getKey(), convertJsonToDagNode(allNodes, dagNode.getValue()));
+ }
+ return new WorkflowDag(findRootNodes(allNodes));
+ }
+
+ /**
+ * Converts {@link WorkflowDag} to a YAML representation.
+ */
+ public static String convertDagToYaml(WorkflowDag dag) throws IOException {
+ final ObjectMapper yamlWriter = new ObjectMapper(new YAMLFactory().disable(Feature.WRITE_DOC_START_MARKER)
+ .enable(Feature.MINIMIZE_QUOTES).enable(JsonParser.Feature.ALLOW_UNQUOTED_FIELD_NAMES));
+ JsonNode yamlNode = MAPPER.createObjectNode();
+ convertDagToYaml(yamlNode, dag.getNodeList());
+ return yamlWriter.writerWithDefaultPrettyPrinter().writeValueAsString(yamlNode);
+ }
+
+ private static void convertDagToYaml(JsonNode yamlNode, List dagNodes) throws IOException {
+ for (DagNode dagNode : dagNodes) {
+ String name = dagNode.getConfig().getOtherConfigs().getOrDefault(DeltaConfig.Config.NODE_NAME, dagNode.getName()).toString();
+ ((ObjectNode) yamlNode).put(name, convertDagNodeToJsonNode(dagNode));
+ if (dagNode.getChildNodes().size() > 0) {
+ convertDagToYaml(yamlNode, dagNode.getChildNodes());
+ }
+ }
+ }
+
+ private static DagNode convertJsonToDagNode(Map allNodes, JsonNode node) throws IOException {
+ String type = node.get(DeltaConfig.Config.TYPE).asText();
+ final DagNode retNode = convertJsonToDagNode(node, type);
+ Arrays.asList(node.get(DeltaConfig.Config.DEPENDENCIES).textValue().split(",")).stream().forEach(dep -> {
+ DagNode parentNode = allNodes.get(dep);
+ if (parentNode != null) {
+ parentNode.addChildNode(retNode);
+ }
+ });
+ return retNode;
+ }
+
+ private static List findRootNodes(Map allNodes) {
+ final List rootNodes = new ArrayList<>();
+ allNodes.entrySet().stream().forEach(entry -> {
+ if (entry.getValue().getParentNodes().size() < 1) {
+ rootNodes.add(entry.getValue());
+ }
+ });
+ return rootNodes;
+ }
+
+ private static DagNode convertJsonToDagNode(JsonNode node, String type) {
+ try {
+ DeltaConfig.Config config = DeltaConfig.Config.newBuilder().withConfigsMap(convertJsonNodeToMap(node)).build();
+ return (DagNode) ReflectionUtils.loadClass(generateFQN(type), config);
+ } catch (ClassNotFoundException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ private static String generateFQN(String name) throws ClassNotFoundException {
+ return Class.forName(StringUtils.joinUsingDelim(".",
+ DagNode.class.getName().substring(0, DagNode.class.getName().lastIndexOf(".")), name)).getName();
+ }
+
+ private static JsonNode convertDagNodeToJsonNode(DagNode node) throws IOException {
+ return createJsonNode(node, node.getClass().getSimpleName());
+ }
+
+ private static Map convertJsonNodeToMap(JsonNode node) {
+ Map configsMap = new HashMap<>();
+ Iterator> itr = node.get(DeltaConfig.Config.CONFIG_NAME).fields();
+ while (itr.hasNext()) {
+ Entry entry = itr.next();
+ switch (entry.getKey()) {
+ case DeltaConfig.Config.HIVE_QUERIES:
+ configsMap.put(DeltaConfig.Config.HIVE_QUERIES, getHiveQueries(entry));
+ break;
+ case DeltaConfig.Config.HIVE_PROPERTIES:
+ configsMap.put(DeltaConfig.Config.HIVE_PROPERTIES, getProperties(entry));
+ break;
+ default:
+ configsMap.put(entry.getKey(), getValue(entry.getValue()));
+ break;
+ }
+ }
+ return configsMap;
+ }
+
+ private static List> getHiveQueries(Entry entry) {
+ List> queries = new ArrayList<>();
+ Iterator> queriesItr = entry.getValue().fields();
+ while (queriesItr.hasNext()) {
+ queries.add(Pair.of(queriesItr.next().getValue().textValue(), queriesItr.next().getValue().asInt()));
+ }
+ return queries;
+ }
+
+ private static List getProperties(Entry entry) {
+ List properties = new ArrayList<>();
+ Iterator> queriesItr = entry.getValue().fields();
+ while (queriesItr.hasNext()) {
+ properties.add(queriesItr.next().getValue().textValue());
+ }
+ return properties;
+ }
+
+ private static Object getValue(JsonNode node) {
+ if (node.isInt()) {
+ return node.asInt();
+ } else if (node.isLong()) {
+ return node.asLong();
+ } else if (node.isShort()) {
+ return node.asInt();
+ } else if (node.isBoolean()) {
+ return node.asBoolean();
+ } else if (node.isDouble()) {
+ return node.asDouble();
+ } else if (node.isFloat()) {
+ return node.asDouble();
+ }
+ return node.textValue();
+ }
+
+ private static JsonNode createJsonNode(DagNode node, String type) throws IOException {
+ JsonNode configNode = MAPPER.readTree(node.getConfig().toString());
+ JsonNode jsonNode = MAPPER.createObjectNode();
+ ((ObjectNode) jsonNode).put(DeltaConfig.Config.CONFIG_NAME, configNode);
+ ((ObjectNode) jsonNode).put(DeltaConfig.Config.TYPE, type);
+ ((ObjectNode) jsonNode).put(DeltaConfig.Config.DEPENDENCIES, getDependencyNames(node));
+ return jsonNode;
+ }
+
+ private static String getDependencyNames(DagNode node) {
+ return node.getParentNodes().stream()
+ .map(e -> ((DagNode) e).getConfig().getOtherConfigs().getOrDefault(DeltaConfig.Config.NODE_NAME, node.getName()).toString())
+ .collect(Collectors.joining(",")).toString();
+ }
+
+ public static String toString(InputStream inputStream) throws IOException {
+ ByteArrayOutputStream result = new ByteArrayOutputStream();
+ byte[] buffer = new byte[1024];
+ int length;
+ while ((length = inputStream.read(buffer)) != -1) {
+ result.write(buffer, 0, length);
+ }
+ return result.toString("utf-8");
+ }
+
+}
diff --git a/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/dag/ExecutionContext.java b/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/dag/ExecutionContext.java
new file mode 100644
index 000000000..eecd763d1
--- /dev/null
+++ b/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/dag/ExecutionContext.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.integ.testsuite.dag;
+
+import java.io.Serializable;
+import org.apache.hudi.integ.testsuite.HoodieTestSuiteWriter;
+import org.apache.hudi.integ.testsuite.dag.nodes.DagNode;
+import org.apache.hudi.integ.testsuite.generator.DeltaGenerator;
+import org.apache.spark.api.java.JavaSparkContext;
+
+/**
+ * This wraps the context needed for an execution of
+ * a {@link DagNode#execute(ExecutionContext)}.
+ */
+public class ExecutionContext implements Serializable {
+
+ private HoodieTestSuiteWriter hoodieTestSuiteWriter;
+ private DeltaGenerator deltaGenerator;
+ private transient JavaSparkContext jsc;
+
+ public ExecutionContext(JavaSparkContext jsc, HoodieTestSuiteWriter hoodieTestSuiteWriter, DeltaGenerator deltaGenerator) {
+ this.hoodieTestSuiteWriter = hoodieTestSuiteWriter;
+ this.deltaGenerator = deltaGenerator;
+ this.jsc = jsc;
+ }
+
+ public HoodieTestSuiteWriter getHoodieTestSuiteWriter() {
+ return hoodieTestSuiteWriter;
+ }
+
+ public DeltaGenerator getDeltaGenerator() {
+ return deltaGenerator;
+ }
+
+ public JavaSparkContext getJsc() {
+ return jsc;
+ }
+}
diff --git a/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/dag/SimpleWorkflowDagGenerator.java b/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/dag/SimpleWorkflowDagGenerator.java
new file mode 100644
index 000000000..ad6e9cb0c
--- /dev/null
+++ b/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/dag/SimpleWorkflowDagGenerator.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.integ.testsuite.dag;
+
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.hudi.common.util.collection.Pair;
+import org.apache.hudi.integ.testsuite.configuration.DeltaConfig;
+import org.apache.hudi.integ.testsuite.dag.nodes.DagNode;
+import org.apache.hudi.integ.testsuite.dag.nodes.HiveQueryNode;
+import org.apache.hudi.integ.testsuite.dag.nodes.InsertNode;
+import org.apache.hudi.integ.testsuite.dag.nodes.UpsertNode;
+
+/**
+ * An example of how to generate a workflow dag programmatically. This is also used as the default workflow dag if
+ * none is provided.
+ */
+public class SimpleWorkflowDagGenerator implements WorkflowDagGenerator {
+
+ @Override
+ public WorkflowDag build() {
+
+ DagNode root = new InsertNode(DeltaConfig.Config.newBuilder()
+ .withNumRecordsToInsert(100)
+ .withNumInsertPartitions(1)
+ .withNumTimesToRepeat(2)
+ .withRecordSize(1000).build());
+
+ DagNode child1 = new InsertNode(DeltaConfig.Config.newBuilder()
+ .withNumRecordsToInsert(100)
+ .withNumInsertPartitions(1)
+ .withNumTimesToRepeat(2)
+ .withRecordSize(1000).build());
+
+ root.addChildNode(child1);
+
+ DagNode child1OfChild1 = new UpsertNode(DeltaConfig.Config.newBuilder()
+ .withNumRecordsToUpdate(100)
+ .withNumUpsertPartitions(2)
+ .withNumTimesToRepeat(1)
+ .withRecordSize(1000).build());
+
+ // Tests running 2 nodes in parallel
+ child1.addChildNode(child1OfChild1);
+
+ List> queryAndResult = new ArrayList<>();
+ queryAndResult.add(Pair.of("select " + "count(*) from testdb1.table1 group "
+ + "by rider having count(*) < 1", 0));
+ DagNode child2OfChild1 = new HiveQueryNode(DeltaConfig.Config.newBuilder()
+ .withHiveQueryAndResults(queryAndResult).withHiveLocal(true).build());
+ child1.addChildNode(child2OfChild1);
+
+ List rootNodes = new ArrayList<>();
+ rootNodes.add(root);
+
+ return new WorkflowDag(rootNodes);
+ }
+
+}
diff --git a/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/dag/WorkflowDag.java b/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/dag/WorkflowDag.java
new file mode 100644
index 000000000..e9171fc47
--- /dev/null
+++ b/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/dag/WorkflowDag.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.integ.testsuite.dag;
+
+import java.util.List;
+import org.apache.hudi.integ.testsuite.dag.nodes.DagNode;
+
+/**
+ * Workflow dag that encapsulates all execute nodes.
+ */
+public class WorkflowDag {
+
+ private List> nodeList;
+
+ public WorkflowDag(List> nodeList) {
+ this.nodeList = nodeList;
+ }
+
+ public List> getNodeList() {
+ return nodeList;
+ }
+
+}
diff --git a/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/dag/WorkflowDagGenerator.java b/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/dag/WorkflowDagGenerator.java
new file mode 100644
index 000000000..98b086e4f
--- /dev/null
+++ b/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/dag/WorkflowDagGenerator.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.integ.testsuite.dag;
+
+/**
+ * A interface which represents a workflow dag generator.
+ */
+public interface WorkflowDagGenerator {
+
+ /**
+ * Builds a {@link WorkflowDag}.
+ */
+ WorkflowDag build();
+
+}
diff --git a/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/dag/nodes/BulkInsertNode.java b/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/dag/nodes/BulkInsertNode.java
new file mode 100644
index 000000000..7a8f40550
--- /dev/null
+++ b/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/dag/nodes/BulkInsertNode.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.integ.testsuite.dag.nodes;
+
+import org.apache.hudi.client.WriteStatus;
+import org.apache.hudi.common.util.Option;
+import org.apache.hudi.integ.testsuite.HoodieTestSuiteWriter;
+import org.apache.hudi.integ.testsuite.configuration.DeltaConfig.Config;
+import org.apache.spark.api.java.JavaRDD;
+
+public class BulkInsertNode extends InsertNode {
+
+ public BulkInsertNode(Config config) {
+ super(config);
+ }
+
+ @Override
+ protected JavaRDD ingest(HoodieTestSuiteWriter hoodieTestSuiteWriter, Option commitTime)
+ throws Exception {
+ log.info("Execute bulk ingest node {}", this.getName());
+ return hoodieTestSuiteWriter.bulkInsert(commitTime);
+ }
+
+}
diff --git a/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/dag/nodes/CleanNode.java b/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/dag/nodes/CleanNode.java
new file mode 100644
index 000000000..7083b47cb
--- /dev/null
+++ b/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/dag/nodes/CleanNode.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.integ.testsuite.dag.nodes;
+
+import org.apache.hudi.integ.testsuite.dag.ExecutionContext;
+
+public class CleanNode extends DagNode {
+
+ public CleanNode() {
+ }
+
+ @Override
+ public void execute(ExecutionContext executionContext) throws Exception {
+ log.info("Executing clean node {}", this.getName());
+ executionContext.getHoodieTestSuiteWriter().getWriteClient(this).clean();
+ }
+
+}
diff --git a/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/dag/nodes/CompactNode.java b/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/dag/nodes/CompactNode.java
new file mode 100644
index 000000000..92fe53c3e
--- /dev/null
+++ b/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/dag/nodes/CompactNode.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.integ.testsuite.dag.nodes;
+
+import org.apache.hudi.client.WriteStatus;
+import org.apache.hudi.common.table.HoodieTableMetaClient;
+import org.apache.hudi.common.table.timeline.HoodieInstant;
+import org.apache.hudi.common.util.Option;
+import org.apache.hudi.integ.testsuite.configuration.DeltaConfig.Config;
+import org.apache.hudi.integ.testsuite.dag.ExecutionContext;
+import org.apache.spark.api.java.JavaRDD;
+
+public class CompactNode extends DagNode> {
+
+ public CompactNode(Config config) {
+ this.config = config;
+ }
+
+ @Override
+ public void execute(ExecutionContext executionContext) throws Exception {
+ HoodieTableMetaClient metaClient = new HoodieTableMetaClient(executionContext.getHoodieTestSuiteWriter().getConfiguration(),
+ executionContext.getHoodieTestSuiteWriter().getCfg().targetBasePath);
+ Option lastInstant = metaClient.getActiveTimeline()
+ .getCommitsAndCompactionTimeline().filterPendingCompactionTimeline().lastInstant();
+ if (lastInstant.isPresent()) {
+ log.info("Compacting instant {}", lastInstant.get());
+ this.result = executionContext.getHoodieTestSuiteWriter().compact(Option.of(lastInstant.get().getTimestamp()));
+ }
+ }
+
+}
diff --git a/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/dag/nodes/DagNode.java b/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/dag/nodes/DagNode.java
new file mode 100644
index 000000000..aa54dc913
--- /dev/null
+++ b/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/dag/nodes/DagNode.java
@@ -0,0 +1,125 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.integ.testsuite.dag.nodes;
+
+import java.util.ArrayList;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Objects;
+import java.util.UUID;
+import org.apache.hudi.integ.testsuite.configuration.DeltaConfig.Config;
+import org.apache.hudi.integ.testsuite.dag.ExecutionContext;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Represents a Node in the DAG of operations for a workflow.
+ */
+public abstract class DagNode implements Comparable> {
+
+ protected static Logger log = LoggerFactory.getLogger(DagNode.class);
+
+ protected List> childNodes;
+ protected List> parentNodes;
+ protected O result;
+ protected Config config;
+ private boolean isCompleted;
+
+ public DagNode addChildNode(DagNode childNode) {
+ childNode.getParentNodes().add(this);
+ getChildNodes().add(childNode);
+ return this;
+ }
+
+ public DagNode addParentNode(DagNode parentNode) {
+ if (!this.getParentNodes().contains(parentNode)) {
+ this.getParentNodes().add(parentNode);
+ }
+ return this;
+ }
+
+ public O getResult() {
+ return result;
+ }
+
+ public List> getChildNodes() {
+ if (childNodes == null) {
+ childNodes = new LinkedList<>();
+ }
+ return childNodes;
+ }
+
+ public List> getParentNodes() {
+ if (parentNodes == null) {
+ this.parentNodes = new ArrayList<>();
+ }
+ return this.parentNodes;
+ }
+
+ public void setParentNodes(List> parentNodes) {
+ this.parentNodes = parentNodes;
+ }
+
+ public abstract void execute(ExecutionContext context) throws Exception;
+
+ public boolean isCompleted() {
+ return isCompleted;
+ }
+
+ public void setCompleted(boolean completed) {
+ isCompleted = completed;
+ }
+
+ public Config getConfig() {
+ return config;
+ }
+
+ public String getName() {
+ Object name = this.config.getOtherConfigs().get(Config.NODE_NAME);
+ if (name == null) {
+ String randomName = UUID.randomUUID().toString();
+ this.config.getOtherConfigs().put(Config.NODE_NAME, randomName);
+ return randomName;
+ }
+ return name.toString();
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ DagNode dagNode = (DagNode) o;
+ return getName() == dagNode.getName();
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(getName());
+ }
+
+ @Override
+ public int compareTo(DagNode thatNode) {
+ return this.hashCode() - thatNode.hashCode();
+ }
+
+}
diff --git a/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/dag/nodes/HiveQueryNode.java b/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/dag/nodes/HiveQueryNode.java
new file mode 100644
index 000000000..04f8c2e73
--- /dev/null
+++ b/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/dag/nodes/HiveQueryNode.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.integ.testsuite.dag.nodes;
+
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+import org.apache.hudi.DataSourceUtils;
+import org.apache.hudi.common.util.collection.Pair;
+import org.apache.hudi.hive.HiveSyncConfig;
+import org.apache.hudi.integ.testsuite.configuration.DeltaConfig;
+import org.apache.hudi.integ.testsuite.dag.ExecutionContext;
+import org.apache.hudi.integ.testsuite.helpers.HiveServiceProvider;
+
+public class HiveQueryNode extends DagNode {
+
+ private HiveServiceProvider hiveServiceProvider;
+
+ public HiveQueryNode(DeltaConfig.Config config) {
+ this.config = config;
+ this.hiveServiceProvider = new HiveServiceProvider(config);
+ }
+
+ @Override
+ public void execute(ExecutionContext executionContext) throws Exception {
+ log.info("Executing hive query node {}", this.getName());
+ this.hiveServiceProvider.startLocalHiveServiceIfNeeded(executionContext.getHoodieTestSuiteWriter().getConfiguration());
+ HiveSyncConfig hiveSyncConfig = DataSourceUtils
+ .buildHiveSyncConfig(executionContext.getHoodieTestSuiteWriter().getDeltaStreamerWrapper()
+ .getDeltaSyncService().getDeltaSync().getProps(),
+ executionContext.getHoodieTestSuiteWriter().getDeltaStreamerWrapper()
+ .getDeltaSyncService().getDeltaSync().getCfg().targetBasePath,
+ executionContext.getHoodieTestSuiteWriter().getDeltaStreamerWrapper()
+ .getDeltaSyncService().getDeltaSync().getCfg().baseFileFormat);
+ this.hiveServiceProvider.syncToLocalHiveIfNeeded(executionContext.getHoodieTestSuiteWriter());
+ Connection con = DriverManager.getConnection(hiveSyncConfig.jdbcUrl, hiveSyncConfig.hiveUser,
+ hiveSyncConfig.hivePass);
+ Statement stmt = con.createStatement();
+ stmt.execute("set hive.input.format=org.apache.hadoop.hive.ql.io.HiveInputFormat");
+ for (String hiveProperty : this.config.getHiveProperties()) {
+ executeStatement(hiveProperty, stmt);
+ }
+ for (Pair queryAndResult : this.config.getHiveQueries()) {
+ log.info("Running {}", queryAndResult.getLeft());
+ ResultSet res = stmt.executeQuery(queryAndResult.getLeft());
+ if (!res.next()) {
+ log.info("res.next() was False - typically this means the query returned no rows.");
+ assert 0 == queryAndResult.getRight();
+ } else {
+ Integer result = res.getInt(1);
+ if (!queryAndResult.getRight().equals(result)) {
+ throw new AssertionError(
+ "QUERY: " + queryAndResult.getLeft()
+ + " | EXPECTED RESULT = " + queryAndResult.getRight()
+ + " | ACTUAL RESULT = " + result
+ );
+ }
+ }
+ log.info("Successfully validated query!");
+ }
+ this.hiveServiceProvider.stopLocalHiveServiceIfNeeded();
+ }
+
+ private void executeStatement(String query, Statement stmt) throws SQLException {
+ log.info("Executing statement {}", stmt.toString());
+ stmt.execute(query);
+ }
+
+}
diff --git a/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/dag/nodes/HiveSyncNode.java b/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/dag/nodes/HiveSyncNode.java
new file mode 100644
index 000000000..f24da1496
--- /dev/null
+++ b/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/dag/nodes/HiveSyncNode.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.integ.testsuite.dag.nodes;
+
+import org.apache.hudi.integ.testsuite.helpers.HiveServiceProvider;
+import org.apache.hudi.integ.testsuite.configuration.DeltaConfig.Config;
+import org.apache.hudi.integ.testsuite.dag.ExecutionContext;
+
+public class HiveSyncNode extends DagNode {
+
+ private HiveServiceProvider hiveServiceProvider;
+
+ public HiveSyncNode(Config config) {
+ this.config = config;
+ this.hiveServiceProvider = new HiveServiceProvider(config);
+ }
+
+ @Override
+ public void execute(ExecutionContext executionContext) throws Exception {
+ log.info("Executing hive sync node");
+ this.hiveServiceProvider.startLocalHiveServiceIfNeeded(executionContext.getHoodieTestSuiteWriter().getConfiguration());
+ this.hiveServiceProvider.syncToLocalHiveIfNeeded(executionContext.getHoodieTestSuiteWriter());
+ executionContext.getHoodieTestSuiteWriter().getDeltaStreamerWrapper().getDeltaSyncService().getDeltaSync().syncHive();
+ this.hiveServiceProvider.stopLocalHiveServiceIfNeeded();
+ }
+
+ public HiveServiceProvider getHiveServiceProvider() {
+ return hiveServiceProvider;
+ }
+}
diff --git a/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/dag/nodes/InsertNode.java b/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/dag/nodes/InsertNode.java
new file mode 100644
index 000000000..23cb28574
--- /dev/null
+++ b/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/dag/nodes/InsertNode.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.integ.testsuite.dag.nodes;
+
+import org.apache.hudi.client.WriteStatus;
+import org.apache.hudi.common.util.Option;
+import org.apache.hudi.integ.testsuite.HoodieTestSuiteWriter;
+import org.apache.hudi.integ.testsuite.configuration.DeltaConfig.Config;
+import org.apache.hudi.integ.testsuite.dag.ExecutionContext;
+import org.apache.hudi.integ.testsuite.generator.DeltaGenerator;
+import org.apache.spark.api.java.JavaRDD;
+
+public class InsertNode extends DagNode> {
+
+ public InsertNode(Config config) {
+ this.config = config;
+ }
+
+ @Override
+ public void execute(ExecutionContext executionContext) throws Exception {
+ generate(executionContext.getDeltaGenerator());
+ log.info("Configs : {}", this.config);
+ if (!config.isDisableIngest()) {
+ log.info("Inserting input data {}", this.getName());
+ Option commitTime = executionContext.getHoodieTestSuiteWriter().startCommit();
+ JavaRDD writeStatus = ingest(executionContext.getHoodieTestSuiteWriter(), commitTime);
+ executionContext.getHoodieTestSuiteWriter().commit(writeStatus, commitTime);
+ this.result = writeStatus;
+ }
+ }
+
+ protected void generate(DeltaGenerator deltaGenerator) throws Exception {
+ if (!config.isDisableGenerate()) {
+ log.info("Generating input data for node {}", this.getName());
+ deltaGenerator.writeRecords(deltaGenerator.generateInserts(config)).count();
+ }
+ }
+
+ protected JavaRDD ingest(HoodieTestSuiteWriter hoodieTestSuiteWriter,
+ Option commitTime) throws Exception {
+ return hoodieTestSuiteWriter.insert(commitTime);
+ }
+
+}
diff --git a/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/dag/nodes/RollbackNode.java b/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/dag/nodes/RollbackNode.java
new file mode 100644
index 000000000..b6d828a10
--- /dev/null
+++ b/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/dag/nodes/RollbackNode.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.integ.testsuite.dag.nodes;
+
+import org.apache.hudi.common.table.HoodieTableMetaClient;
+import org.apache.hudi.common.table.timeline.HoodieInstant;
+import org.apache.hudi.common.util.Option;
+import org.apache.hudi.integ.testsuite.configuration.DeltaConfig.Config;
+import org.apache.hudi.integ.testsuite.dag.ExecutionContext;
+
+public class RollbackNode extends DagNode> {
+
+ public RollbackNode(Config config) {
+ this.config = config;
+ }
+
+ @Override
+ public void execute(ExecutionContext executionContext) throws Exception {
+ log.info("Executing rollback node {}", this.getName());
+ // Can only be done with an instantiation of a new WriteClient hence cannot be done during DeltaStreamer
+ // testing for now
+ HoodieTableMetaClient metaClient = new HoodieTableMetaClient(executionContext.getHoodieTestSuiteWriter().getConfiguration(),
+ executionContext.getHoodieTestSuiteWriter().getCfg().targetBasePath);
+ Option lastInstant = metaClient.getActiveTimeline().getCommitsTimeline().lastInstant();
+ if (lastInstant.isPresent()) {
+ log.info("Rolling back last instant {}", lastInstant.get());
+ executionContext.getHoodieTestSuiteWriter().getWriteClient(this).rollback(lastInstant.get().getTimestamp());
+ this.result = lastInstant;
+ }
+ }
+
+}
diff --git a/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/dag/nodes/ScheduleCompactNode.java b/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/dag/nodes/ScheduleCompactNode.java
new file mode 100644
index 000000000..93502aee1
--- /dev/null
+++ b/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/dag/nodes/ScheduleCompactNode.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.integ.testsuite.dag.nodes;
+
+import org.apache.hudi.common.model.HoodieCommitMetadata;
+import org.apache.hudi.common.table.HoodieTableMetaClient;
+import org.apache.hudi.common.table.timeline.HoodieInstant;
+import org.apache.hudi.common.util.Option;
+import org.apache.hudi.integ.testsuite.configuration.DeltaConfig.Config;
+import org.apache.hudi.integ.testsuite.dag.ExecutionContext;
+
+public class ScheduleCompactNode extends DagNode> {
+
+ public ScheduleCompactNode(Config config) {
+ this.config = config;
+ }
+
+ @Override
+ public void execute(ExecutionContext executionContext) throws Exception {
+ log.info("Executing schedule compact node {}", this.getName());
+ // Can only be done with an instantiation of a new WriteClient hence cannot be done during DeltaStreamer
+ // testing for now
+ // Find the last commit and extra the extra metadata to be passed to the schedule compaction. This is
+ // done to ensure the CHECKPOINT is correctly passed from commit to commit
+ HoodieTableMetaClient metaClient = new HoodieTableMetaClient(executionContext.getHoodieTestSuiteWriter().getConfiguration(),
+ executionContext.getHoodieTestSuiteWriter().getCfg().targetBasePath);
+ Option lastInstant = metaClient.getActiveTimeline().getCommitsTimeline().lastInstant();
+ if (lastInstant.isPresent()) {
+ HoodieCommitMetadata metadata = org.apache.hudi.common.model.HoodieCommitMetadata.fromBytes(metaClient
+ .getActiveTimeline().getInstantDetails(lastInstant.get()).get(), HoodieCommitMetadata.class);
+ Option scheduledInstant = executionContext.getHoodieTestSuiteWriter().scheduleCompaction(Option.of(metadata
+ .getExtraMetadata()));
+ if (scheduledInstant.isPresent()) {
+ log.info("Scheduling compaction instant {}", scheduledInstant.get());
+ }
+ this.result = scheduledInstant;
+ }
+ }
+
+}
diff --git a/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/dag/nodes/SparkSQLQueryNode.java b/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/dag/nodes/SparkSQLQueryNode.java
new file mode 100644
index 000000000..a8a1b72bb
--- /dev/null
+++ b/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/dag/nodes/SparkSQLQueryNode.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.integ.testsuite.dag.nodes;
+
+import org.apache.hudi.common.util.collection.Pair;
+import org.apache.hudi.integ.testsuite.helpers.HiveServiceProvider;
+import org.apache.hudi.integ.testsuite.configuration.DeltaConfig.Config;
+import org.apache.hudi.integ.testsuite.dag.ExecutionContext;
+import org.apache.spark.sql.Dataset;
+import org.apache.spark.sql.Row;
+import org.apache.spark.sql.SparkSession;
+
+public class SparkSQLQueryNode extends DagNode {
+
+ HiveServiceProvider hiveServiceProvider;
+
+ public SparkSQLQueryNode(Config config) {
+ this.config = config;
+ this.hiveServiceProvider = new HiveServiceProvider(config);
+ }
+
+ @Override
+ public void execute(ExecutionContext executionContext) throws Exception {
+ log.info("Executing spark sql query node");
+ this.hiveServiceProvider.startLocalHiveServiceIfNeeded(executionContext.getHoodieTestSuiteWriter().getConfiguration());
+ this.hiveServiceProvider.syncToLocalHiveIfNeeded(executionContext.getHoodieTestSuiteWriter());
+ SparkSession session = SparkSession.builder().sparkContext(executionContext.getJsc().sc()).getOrCreate();
+ for (String hiveProperty : this.config.getHiveProperties()) {
+ session.sql(hiveProperty).count();
+ }
+ for (Pair queryAndResult : this.config.getHiveQueries()) {
+ log.info("Running {}", queryAndResult.getLeft());
+ Dataset res = session.sql(queryAndResult.getLeft());
+ if (res.count() == 0) {
+ assert 0 == queryAndResult.getRight();
+ } else {
+ assert ((Row[]) res.collect())[0].getInt(0) == queryAndResult.getRight();
+ }
+ log.info("Successfully validated query!");
+ }
+ this.hiveServiceProvider.stopLocalHiveServiceIfNeeded();
+ this.result = true;
+ }
+
+}
diff --git a/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/dag/nodes/UpsertNode.java b/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/dag/nodes/UpsertNode.java
new file mode 100644
index 000000000..3872f767b
--- /dev/null
+++ b/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/dag/nodes/UpsertNode.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.integ.testsuite.dag.nodes;
+
+import org.apache.hudi.client.WriteStatus;
+import org.apache.hudi.common.util.Option;
+import org.apache.hudi.integ.testsuite.HoodieTestSuiteWriter;
+import org.apache.hudi.integ.testsuite.configuration.DeltaConfig.Config;
+import org.apache.hudi.integ.testsuite.generator.DeltaGenerator;
+import org.apache.spark.api.java.JavaRDD;
+
+public class UpsertNode extends InsertNode {
+
+ public UpsertNode(Config config) {
+ super(config);
+ }
+
+ @Override
+ protected void generate(DeltaGenerator deltaGenerator) throws Exception {
+ if (!config.isDisableGenerate()) {
+ log.info("Generating input data {}", this.getName());
+ deltaGenerator.writeRecords(deltaGenerator.generateUpdates(config)).count();
+ }
+ }
+
+ @Override
+ protected JavaRDD ingest(HoodieTestSuiteWriter hoodieTestSuiteWriter, Option commitTime)
+ throws Exception {
+ if (!config.isDisableIngest()) {
+ log.info("Upserting input data {}", this.getName());
+ this.result = hoodieTestSuiteWriter.upsert(commitTime);
+ }
+ return this.result;
+ }
+
+}
diff --git a/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/dag/nodes/ValidateNode.java b/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/dag/nodes/ValidateNode.java
new file mode 100644
index 000000000..5ba0249a6
--- /dev/null
+++ b/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/dag/nodes/ValidateNode.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.integ.testsuite.dag.nodes;
+
+import java.util.List;
+import java.util.function.Function;
+import org.apache.hudi.integ.testsuite.configuration.DeltaConfig.Config;
+import org.apache.hudi.integ.testsuite.dag.ExecutionContext;
+
+public class ValidateNode extends DagNode {
+
+ protected Function, R> function;
+
+ public ValidateNode(Config config, Function, R> function) {
+ this.function = function;
+ this.config = config;
+ }
+
+ @Override
+ public void execute(ExecutionContext executionContext) {
+ if (this.getParentNodes().size() > 0 && (Boolean) this.config.getOtherConfigs().getOrDefault("WAIT_FOR_PARENTS",
+ true)) {
+ for (DagNode node : (List) this.getParentNodes()) {
+ if (!node.isCompleted()) {
+ throw new RuntimeException("cannot validate before parent nodes are complete");
+ }
+ }
+ }
+ this.result = this.function.apply((List) this.getParentNodes());
+ }
+
+}
diff --git a/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/dag/scheduler/DagScheduler.java b/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/dag/scheduler/DagScheduler.java
new file mode 100644
index 000000000..7d78d8311
--- /dev/null
+++ b/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/dag/scheduler/DagScheduler.java
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.integ.testsuite.dag.scheduler;
+
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.PriorityQueue;
+import java.util.Queue;
+import java.util.Set;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import org.apache.hudi.exception.HoodieException;
+import org.apache.hudi.integ.testsuite.dag.nodes.DagNode;
+import org.apache.hudi.integ.testsuite.HoodieTestSuiteWriter;
+import org.apache.hudi.integ.testsuite.dag.ExecutionContext;
+import org.apache.hudi.integ.testsuite.dag.WorkflowDag;
+import org.apache.hudi.integ.testsuite.generator.DeltaGenerator;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class DagScheduler {
+
+ private static Logger log = LoggerFactory.getLogger(DagScheduler.class);
+ private WorkflowDag workflowDag;
+ private ExecutionContext executionContext;
+
+ public DagScheduler(WorkflowDag workflowDag, HoodieTestSuiteWriter hoodieTestSuiteWriter, DeltaGenerator deltaGenerator) {
+ this.workflowDag = workflowDag;
+ this.executionContext = new ExecutionContext(null, hoodieTestSuiteWriter, deltaGenerator);
+ }
+
+ public void schedule() throws Exception {
+ ExecutorService service = Executors.newFixedThreadPool(2);
+ try {
+ execute(service, workflowDag.getNodeList());
+ service.shutdown();
+ } finally {
+ if (!service.isShutdown()) {
+ log.info("Forcing shutdown of executor service, this might kill running tasks");
+ service.shutdownNow();
+ }
+ }
+ }
+
+ private void execute(ExecutorService service, List nodes) throws Exception {
+ // Nodes at the same level are executed in parallel
+ Queue queue = new PriorityQueue<>(nodes);
+ log.info("Running workloads");
+ do {
+ List futures = new ArrayList<>();
+ Set childNodes = new HashSet<>();
+ while (queue.size() > 0) {
+ DagNode nodeToExecute = queue.poll();
+ futures.add(service.submit(() -> executeNode(nodeToExecute)));
+ if (nodeToExecute.getChildNodes().size() > 0) {
+ childNodes.addAll(nodeToExecute.getChildNodes());
+ }
+ }
+ queue.addAll(childNodes);
+ childNodes.clear();
+ for (Future future : futures) {
+ future.get(1, TimeUnit.HOURS);
+ }
+ } while (queue.size() > 0);
+ log.info("Finished workloads");
+ }
+
+ private void executeNode(DagNode node) {
+ if (node.isCompleted()) {
+ throw new RuntimeException("DagNode already completed! Cannot re-execute");
+ }
+ try {
+ node.execute(executionContext);
+ node.setCompleted(true);
+ log.info("Finished executing {}", node.getName());
+ } catch (Exception e) {
+ log.error("Exception executing node");
+ throw new HoodieException(e);
+ }
+ }
+}
diff --git a/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/generator/DeltaGenerator.java b/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/generator/DeltaGenerator.java
new file mode 100644
index 000000000..24311e0fe
--- /dev/null
+++ b/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/generator/DeltaGenerator.java
@@ -0,0 +1,237 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.integ.testsuite.generator;
+
+import java.io.IOException;
+import java.io.Serializable;
+import java.io.UncheckedIOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.StreamSupport;
+import org.apache.avro.generic.GenericRecord;
+import org.apache.hudi.common.util.Option;
+import org.apache.hudi.integ.testsuite.converter.Converter;
+import org.apache.hudi.integ.testsuite.converter.UpdateConverter;
+import org.apache.hudi.integ.testsuite.reader.DFSAvroDeltaInputReader;
+import org.apache.hudi.integ.testsuite.reader.DFSHoodieDatasetInputReader;
+import org.apache.hudi.integ.testsuite.reader.DeltaInputReader;
+import org.apache.hudi.integ.testsuite.writer.DeltaOutputMode;
+import org.apache.hudi.integ.testsuite.writer.DeltaWriteStats;
+import org.apache.hudi.integ.testsuite.writer.DeltaWriterAdapter;
+import org.apache.hudi.integ.testsuite.writer.DeltaWriterFactory;
+import org.apache.hudi.keygen.ComplexKeyGenerator;
+import org.apache.hudi.keygen.KeyGenerator;
+import org.apache.hudi.keygen.SimpleKeyGenerator;
+import org.apache.hudi.integ.testsuite.configuration.DFSDeltaConfig;
+import org.apache.hudi.integ.testsuite.configuration.DeltaConfig;
+import org.apache.hudi.integ.testsuite.configuration.DeltaConfig.Config;
+import org.apache.spark.api.java.JavaRDD;
+import org.apache.spark.api.java.JavaSparkContext;
+import org.apache.spark.sql.SparkSession;
+import org.apache.spark.storage.StorageLevel;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import scala.Tuple2;
+
+/**
+ * The delta generator generates all types of workloads (insert, update) for the given configs.
+ */
+public class DeltaGenerator implements Serializable {
+
+ private static Logger log = LoggerFactory.getLogger(DFSHoodieDatasetInputReader.class);
+
+ private DeltaConfig deltaOutputConfig;
+ private transient JavaSparkContext jsc;
+ private transient SparkSession sparkSession;
+ private String schemaStr;
+ private List recordRowKeyFieldNames;
+ private List partitionPathFieldNames;
+ private int batchId;
+
+ public DeltaGenerator(DeltaConfig deltaOutputConfig, JavaSparkContext jsc, SparkSession sparkSession,
+ String schemaStr,
+ KeyGenerator keyGenerator) {
+ this.deltaOutputConfig = deltaOutputConfig;
+ this.jsc = jsc;
+ this.sparkSession = sparkSession;
+ this.schemaStr = schemaStr;
+ this.recordRowKeyFieldNames = keyGenerator instanceof ComplexKeyGenerator ? ((ComplexKeyGenerator) keyGenerator)
+ .getRecordKeyFields() : Arrays.asList(((SimpleKeyGenerator) keyGenerator).getRecordKeyField());
+ this.partitionPathFieldNames = keyGenerator instanceof ComplexKeyGenerator ? ((ComplexKeyGenerator) keyGenerator)
+ .getPartitionPathFields() : Arrays.asList(((SimpleKeyGenerator) keyGenerator).getPartitionPathField());
+ }
+
+ public JavaRDD writeRecords(JavaRDD records) {
+ // The following creates a new anonymous function for iterator and hence results in serialization issues
+ JavaRDD ws = records.mapPartitions(itr -> {
+ try {
+ DeltaWriterAdapter deltaWriterAdapter = DeltaWriterFactory
+ .getDeltaWriterAdapter(deltaOutputConfig, batchId);
+ return Collections.singletonList(deltaWriterAdapter.write(itr)).iterator();
+ } catch (IOException io) {
+ throw new UncheckedIOException(io);
+ }
+ }).flatMap(List::iterator);
+ batchId++;
+ return ws;
+ }
+
+ public JavaRDD generateInserts(Config operation) {
+ long recordsPerPartition = operation.getNumRecordsInsert();
+ int minPayloadSize = operation.getRecordSize();
+ JavaRDD inputBatch = jsc.parallelize(Collections.EMPTY_LIST)
+ .repartition(operation.getNumInsertPartitions()).mapPartitions(p -> {
+ return new LazyRecordGeneratorIterator(new FlexibleSchemaRecordGenerationIterator(recordsPerPartition,
+ minPayloadSize, schemaStr, partitionPathFieldNames));
+ });
+ return inputBatch;
+ }
+
+ public JavaRDD generateUpdates(Config config) throws IOException {
+ if (deltaOutputConfig.getDeltaOutputMode() == DeltaOutputMode.DFS) {
+ JavaRDD inserts = null;
+ if (config.getNumRecordsInsert() > 0) {
+ inserts = generateInserts(config);
+ }
+ DeltaInputReader deltaInputReader = null;
+ JavaRDD adjustedRDD = null;
+ if (config.getNumUpsertPartitions() < 1) {
+ // randomly generate updates for a given number of records without regard to partitions and files
+ deltaInputReader = new DFSAvroDeltaInputReader(sparkSession, schemaStr,
+ ((DFSDeltaConfig) deltaOutputConfig).getDeltaBasePath(), Option.empty(), Option.empty());
+ adjustedRDD = deltaInputReader.read(config.getNumRecordsUpsert());
+ adjustedRDD = adjustRDDToGenerateExactNumUpdates(adjustedRDD, jsc, config.getNumRecordsUpsert());
+ } else {
+ deltaInputReader =
+ new DFSHoodieDatasetInputReader(jsc, ((DFSDeltaConfig) deltaOutputConfig).getDatasetOutputPath(),
+ schemaStr);
+ if (config.getFractionUpsertPerFile() > 0) {
+ adjustedRDD = deltaInputReader.read(config.getNumUpsertPartitions(), config.getNumUpsertFiles(),
+ config.getFractionUpsertPerFile());
+ } else {
+ adjustedRDD = deltaInputReader.read(config.getNumUpsertPartitions(), config.getNumUpsertFiles(), config
+ .getNumRecordsUpsert());
+ }
+ }
+ log.info("Repartitioning records");
+ // persist this since we will make multiple passes over this
+ adjustedRDD = adjustedRDD.repartition(jsc.defaultParallelism());
+ log.info("Repartitioning records done");
+ Converter converter = new UpdateConverter(schemaStr, config.getRecordSize(),
+ partitionPathFieldNames, recordRowKeyFieldNames);
+ JavaRDD updates = converter.convert(adjustedRDD);
+ log.info("Records converted");
+ updates.persist(StorageLevel.DISK_ONLY());
+ return inserts != null ? inserts.union(updates) : updates;
+ // TODO : Generate updates for only N partitions.
+ } else {
+ throw new IllegalArgumentException("Other formats are not supported at the moment");
+ }
+ }
+
+ public Map getPartitionToCountMap(JavaRDD records) {
+ // Requires us to keep the partitioner the same
+ return records.mapPartitionsWithIndex((index, itr) -> {
+ Iterable newIterable = () -> itr;
+ // parallelize counting for speed
+ long count = StreamSupport.stream(newIterable.spliterator(), true).count();
+ return Arrays.asList(new Tuple2<>(index, count)).iterator();
+ }, true).mapToPair(i -> i).collectAsMap();
+ }
+
+ public Map getAdjustedPartitionsCount(Map partitionCountMap, long
+ recordsToRemove) {
+ long remainingRecordsToRemove = recordsToRemove;
+ Iterator> iterator = partitionCountMap.entrySet().iterator();
+ Map adjustedPartitionCountMap = new HashMap<>();
+ while (iterator.hasNext()) {
+ Map.Entry entry = iterator.next();
+ if (entry.getValue() < remainingRecordsToRemove) {
+ remainingRecordsToRemove -= entry.getValue();
+ adjustedPartitionCountMap.put(entry.getKey(), 0L);
+ } else {
+ long newValue = entry.getValue() - remainingRecordsToRemove;
+ remainingRecordsToRemove = 0;
+ adjustedPartitionCountMap.put(entry.getKey(), newValue);
+ }
+ if (remainingRecordsToRemove == 0) {
+ break;
+ }
+ }
+ return adjustedPartitionCountMap;
+ }
+
+ public JavaRDD adjustRDDToGenerateExactNumUpdates(JavaRDD updates, JavaSparkContext
+ jsc, long totalRecordsRequired) {
+ Map actualPartitionCountMap = getPartitionToCountMap(updates);
+ long totalRecordsGenerated = actualPartitionCountMap.values().stream().mapToLong(Long::longValue).sum();
+ if (isSafeToTake(totalRecordsRequired, totalRecordsGenerated)) {
+ // Generate totalRecordsRequired - totalRecordsGenerated new records and union the RDD's
+ // NOTE : This performs poorly when totalRecordsRequired >> totalRecordsGenerated. Hence, always
+ // ensure that enough inserts are created before hand (this needs to be noted during the WorkflowDag creation)
+ long sizeOfUpdateRDD = totalRecordsGenerated;
+ while (totalRecordsRequired != sizeOfUpdateRDD) {
+ long recordsToTake = (totalRecordsRequired - sizeOfUpdateRDD) > sizeOfUpdateRDD
+ ? sizeOfUpdateRDD : (totalRecordsRequired - sizeOfUpdateRDD);
+ if ((totalRecordsRequired - sizeOfUpdateRDD) > recordsToTake && recordsToTake <= sizeOfUpdateRDD) {
+ updates = updates.union(updates);
+ sizeOfUpdateRDD *= 2;
+ } else {
+ List remainingUpdates = updates.take((int) (recordsToTake));
+ updates = updates.union(jsc.parallelize(remainingUpdates));
+ sizeOfUpdateRDD = sizeOfUpdateRDD + recordsToTake;
+ }
+ }
+ return updates;
+ } else if (totalRecordsRequired < totalRecordsGenerated) {
+ final Map adjustedPartitionCountMap = getAdjustedPartitionsCount(actualPartitionCountMap,
+ totalRecordsGenerated - totalRecordsRequired);
+ // limit counts across partitions to meet the exact number of updates required
+ JavaRDD trimmedRecords = updates.mapPartitionsWithIndex((index, itr) -> {
+ int counter = 1;
+ List entriesToKeep = new ArrayList<>();
+ if (!adjustedPartitionCountMap.containsKey(index)) {
+ return itr;
+ } else {
+ long recordsToKeepForThisPartition = adjustedPartitionCountMap.get(index);
+ while (counter <= recordsToKeepForThisPartition && itr.hasNext()) {
+ entriesToKeep.add(itr.next());
+ counter++;
+ }
+ return entriesToKeep.iterator();
+ }
+ }, true);
+ return trimmedRecords;
+ }
+ return updates;
+ }
+
+ private boolean isSafeToTake(long totalRecords, long totalRecordsGenerated) {
+ // TODO : Ensure that the difference between totalRecords and totalRecordsGenerated is not too big, if yes,
+ // then there are fewer number of records on disk, hence we need to find another way to generate updates when
+ // requiredUpdates >> insertedRecords
+ return totalRecords > totalRecordsGenerated;
+ }
+
+}
diff --git a/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/generator/FlexibleSchemaRecordGenerationIterator.java b/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/generator/FlexibleSchemaRecordGenerationIterator.java
new file mode 100644
index 000000000..614add52c
--- /dev/null
+++ b/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/generator/FlexibleSchemaRecordGenerationIterator.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.integ.testsuite.generator;
+
+import java.util.Iterator;
+import java.util.List;
+import org.apache.avro.Schema;
+import org.apache.avro.generic.GenericRecord;
+
+/**
+ * A GenericRecordGeneratorIterator for the custom schema of the workload. Implements {@link Iterator} to allow for
+ * iteration semantics.
+ */
+public class FlexibleSchemaRecordGenerationIterator implements Iterator {
+
+ // Stores how many records to generate as part of this iterator. Ideally, one iterator is started per spark
+ // partition.
+ private long counter;
+ // Use the full payload generator as default
+ private GenericRecordFullPayloadGenerator generator;
+ // Store last record for the partition path of the first payload to be used for all subsequent generated payloads
+ private GenericRecord lastRecord;
+ // Partition path field name
+ private List partitionPathFieldNames;
+
+ public FlexibleSchemaRecordGenerationIterator(long maxEntriesToProduce, String schema) {
+ this(maxEntriesToProduce, GenericRecordFullPayloadGenerator.DEFAULT_PAYLOAD_SIZE, schema, null);
+ }
+
+ public FlexibleSchemaRecordGenerationIterator(long maxEntriesToProduce, int minPayloadSize, String schemaStr,
+ List partitionPathFieldNames) {
+ this.counter = maxEntriesToProduce;
+ this.partitionPathFieldNames = partitionPathFieldNames;
+ Schema schema = new Schema.Parser().parse(schemaStr);
+ this.generator = new GenericRecordFullPayloadGenerator(schema, minPayloadSize);
+ }
+
+ @Override
+ public boolean hasNext() {
+ return this.counter > 0;
+ }
+
+ @Override
+ public GenericRecord next() {
+ this.counter--;
+ if (lastRecord == null) {
+ GenericRecord record = this.generator.getNewPayload();
+ lastRecord = record;
+ return record;
+ } else {
+ return this.generator.randomize(lastRecord, this.partitionPathFieldNames);
+ }
+ }
+}
diff --git a/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/generator/GenericRecordFullPayloadGenerator.java b/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/generator/GenericRecordFullPayloadGenerator.java
new file mode 100644
index 000000000..df1a4c7cd
--- /dev/null
+++ b/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/generator/GenericRecordFullPayloadGenerator.java
@@ -0,0 +1,285 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.integ.testsuite.generator;
+
+import java.io.Serializable;
+import java.nio.ByteBuffer;
+import java.nio.charset.Charset;
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+import java.util.UUID;
+import org.apache.avro.LogicalType;
+import org.apache.avro.LogicalTypes;
+import org.apache.avro.Schema;
+import org.apache.avro.Schema.Type;
+import org.apache.avro.generic.GenericData;
+import org.apache.avro.generic.GenericData.Fixed;
+import org.apache.avro.generic.GenericFixed;
+import org.apache.avro.generic.GenericRecord;
+import org.apache.hudi.common.util.collection.Pair;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * This is a GenericRecord payload generator that generates full generic records {@link GenericRecord}.
+ * Every field of a generic record created using this generator contains a random value.
+ */
+public class GenericRecordFullPayloadGenerator implements Serializable {
+
+ public static final int DEFAULT_PAYLOAD_SIZE = 1024 * 10; // 10 KB
+ private static Logger log = LoggerFactory.getLogger(GenericRecordFullPayloadGenerator.class);
+ protected final Random random = new Random();
+ // The source schema used to generate a payload
+ private final transient Schema baseSchema;
+ // Used to validate a generic record
+ private final transient GenericData genericData = new GenericData();
+ // Number of more bytes to add based on the estimated full record payload size and min payload size
+ private int numberOfBytesToAdd;
+ // If more elements should be packed to meet the minPayloadSize
+ private boolean shouldAddMore;
+ // How many complex fields have we visited that can help us pack more entries and increase the size of the record
+ private int numberOfComplexFields;
+ // The size of a full record where every field of a generic record created contains 1 random value
+ private int estimatedFullPayloadSize;
+ // LogicalTypes in Avro 1.8.2
+ private static final String DECIMAL = "decimal";
+ private static final String UUID_NAME = "uuid";
+ private static final String DATE = "date";
+ private static final String TIME_MILLIS = "time-millis";
+ private static final String TIME_MICROS = "time-micros";
+ private static final String TIMESTAMP_MILLIS = "timestamp-millis";
+ private static final String TIMESTAMP_MICROS = "timestamp-micros";
+
+ public GenericRecordFullPayloadGenerator(Schema schema) {
+ this(schema, DEFAULT_PAYLOAD_SIZE);
+ }
+
+ public GenericRecordFullPayloadGenerator(Schema schema, int minPayloadSize) {
+ Pair sizeInfo = new GenericRecordFullPayloadSizeEstimator(schema)
+ .typeEstimateAndNumComplexFields();
+ this.estimatedFullPayloadSize = sizeInfo.getLeft();
+ this.numberOfComplexFields = sizeInfo.getRight();
+ this.baseSchema = schema;
+ this.shouldAddMore = estimatedFullPayloadSize < minPayloadSize;
+ if (this.shouldAddMore) {
+ this.numberOfBytesToAdd = minPayloadSize - estimatedFullPayloadSize;
+ if (numberOfComplexFields < 1) {
+ log.warn("The schema does not have any collections/complex fields. Cannot achieve minPayloadSize : {}",
+ minPayloadSize);
+ }
+ }
+ }
+
+ protected static boolean isPrimitive(Schema localSchema) {
+ if (localSchema.getType() != Type.ARRAY
+ && localSchema.getType() != Type.MAP
+ && localSchema.getType() != Type.RECORD
+ && localSchema.getType() != Type.UNION) {
+ return true;
+ } else {
+ return false;
+ }
+ }
+
+ public GenericRecord getNewPayload() {
+ return convert(baseSchema);
+ }
+
+ public GenericRecord getUpdatePayload(GenericRecord record, List blacklistFields) {
+ return randomize(record, blacklistFields);
+ }
+
+ protected GenericRecord convert(Schema schema) {
+ GenericRecord result = new GenericData.Record(schema);
+ for (Schema.Field f : schema.getFields()) {
+ result.put(f.name(), typeConvert(f.schema()));
+ }
+ return result;
+ }
+
+ protected GenericRecord convertPartial(Schema schema) {
+ GenericRecord result = new GenericData.Record(schema);
+ for (Schema.Field f : schema.getFields()) {
+ boolean setNull = random.nextBoolean();
+ if (!setNull) {
+ result.put(f.name(), typeConvert(f.schema()));
+ } else {
+ result.put(f.name(), null);
+ }
+ }
+ // TODO : pack remaining bytes into a complex field
+ return result;
+ }
+
+ protected GenericRecord randomize(GenericRecord record, List blacklistFields) {
+ for (Schema.Field f : record.getSchema().getFields()) {
+ if (blacklistFields == null || !blacklistFields.contains(f.name())) {
+ record.put(f.name(), typeConvert(f.schema()));
+ }
+ }
+ return record;
+ }
+
+ private Object typeConvert(Schema schema) {
+ Schema localSchema = schema;
+ if (isOption(schema)) {
+ localSchema = getNonNull(schema);
+ }
+ switch (localSchema.getType()) {
+ case BOOLEAN:
+ return random.nextBoolean();
+ case DOUBLE:
+ return random.nextDouble();
+ case FLOAT:
+ return random.nextFloat();
+ case INT:
+ return random.nextInt();
+ case LONG:
+ return random.nextLong();
+ case STRING:
+ return UUID.randomUUID().toString();
+ case ENUM:
+ List enumSymbols = localSchema.getEnumSymbols();
+ return new GenericData.EnumSymbol(localSchema, enumSymbols.get(random.nextInt(enumSymbols.size() - 1)));
+ case RECORD:
+ return convert(localSchema);
+ case ARRAY:
+ Schema elementSchema = localSchema.getElementType();
+ List listRes = new ArrayList();
+ if (isPrimitive(elementSchema) && this.shouldAddMore) {
+ int numEntriesToAdd = numEntriesToAdd(elementSchema);
+ while (numEntriesToAdd > 0) {
+ listRes.add(typeConvert(elementSchema));
+ numEntriesToAdd--;
+ }
+ } else {
+ listRes.add(typeConvert(elementSchema));
+ }
+ return listRes;
+ case MAP:
+ Schema valueSchema = localSchema.getValueType();
+ Map mapRes = new HashMap();
+ if (isPrimitive(valueSchema) && this.shouldAddMore) {
+ int numEntriesToAdd = numEntriesToAdd(valueSchema);
+ while (numEntriesToAdd > 0) {
+ mapRes.put(UUID.randomUUID().toString(), typeConvert(valueSchema));
+ numEntriesToAdd--;
+ }
+ } else {
+ mapRes.put(UUID.randomUUID().toString(), typeConvert(valueSchema));
+ }
+ return mapRes;
+ case BYTES:
+ return ByteBuffer.wrap(UUID.randomUUID().toString().getBytes(Charset.defaultCharset()));
+ case FIXED:
+ return generateFixedType(localSchema);
+ default:
+ throw new IllegalArgumentException(
+ "Cannot handle type: " + localSchema.getType());
+ }
+ }
+
+ private Object generateFixedType(Schema localSchema) {
+ // TODO: Need to implement valid data generation for fixed type
+ GenericFixed genericFixed = new GenericData.Fixed(localSchema);
+ switch (localSchema.getLogicalType().getName()) {
+ case UUID_NAME:
+ ((Fixed) genericFixed).bytes(UUID.randomUUID().toString().getBytes());
+ return genericFixed;
+ case DECIMAL:
+ return genericFixed;
+ case DATE:
+ return genericFixed;
+ case TIME_MILLIS:
+ return genericFixed;
+ default:
+ throw new IllegalArgumentException(
+ "Cannot handle type: " + localSchema.getLogicalType());
+ }
+ }
+
+ public boolean validate(GenericRecord record) {
+ return genericData.validate(baseSchema, record);
+ }
+
+ protected boolean isOption(Schema schema) {
+ return schema.getType().equals(Schema.Type.UNION)
+ && schema.getTypes().size() == 2
+ && (schema.getTypes().get(0).getType().equals(Schema.Type.NULL)
+ || schema.getTypes().get(1).getType().equals(Schema.Type.NULL));
+ }
+
+ protected Schema getNonNull(Schema schema) {
+ List types = schema.getTypes();
+ return types.get(0).getType().equals(Schema.Type.NULL) ? types.get(1) : types.get(0);
+ }
+
+ public int getEstimatedFullPayloadSize() {
+ return estimatedFullPayloadSize;
+ }
+
+ private int getSize(Schema elementSchema) {
+ switch (elementSchema.getType()) {
+ case BOOLEAN:
+ return 1;
+ case DOUBLE:
+ return Double.BYTES;
+ case FLOAT:
+ return Float.BYTES;
+ case INT:
+ return Integer.BYTES;
+ case LONG:
+ return Long.BYTES;
+ case STRING:
+ return UUID.randomUUID().toString().length();
+ case ENUM:
+ return 1;
+ case BYTES:
+ return UUID.randomUUID().toString().length();
+ case FIXED:
+ return elementSchema.getFixedSize();
+ default:
+ throw new RuntimeException("Unknown type " + elementSchema.getType());
+ }
+ }
+
+ private int numEntriesToAdd(Schema elementSchema) {
+ // Find the size of the primitive data type in bytes
+ int primitiveDataTypeSize = getSize(elementSchema);
+ int numEntriesToAdd = numberOfBytesToAdd / primitiveDataTypeSize;
+ // If more than 10 entries are being added for this same complex field and there are still more complex fields to
+ // be visited in the schema, reduce the number of entries to add by a factor of 10 to allow for other complex
+ // fields to pack some entries
+ if (numEntriesToAdd % 10 > 0 && this.numberOfComplexFields > 1) {
+ numEntriesToAdd = numEntriesToAdd / 10;
+ numberOfBytesToAdd -= numEntriesToAdd * primitiveDataTypeSize;
+ this.shouldAddMore = true;
+ } else {
+ this.numberOfBytesToAdd = 0;
+ this.shouldAddMore = false;
+ }
+ this.numberOfComplexFields -= 1;
+ return numEntriesToAdd;
+ }
+}
+
diff --git a/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/generator/GenericRecordFullPayloadSizeEstimator.java b/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/generator/GenericRecordFullPayloadSizeEstimator.java
new file mode 100644
index 000000000..c6a8f4e31
--- /dev/null
+++ b/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/generator/GenericRecordFullPayloadSizeEstimator.java
@@ -0,0 +1,121 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.integ.testsuite.generator;
+
+import java.io.Serializable;
+import java.util.List;
+import java.util.UUID;
+import java.util.concurrent.atomic.AtomicInteger;
+import org.apache.avro.Schema;
+import org.apache.avro.generic.GenericRecord;
+import org.apache.hudi.common.util.collection.Pair;
+
+/**
+ * This is a GenericRecord payload estimator estimates the size of a full generic record {@link GenericRecord}.
+ * A full record is defined as "Every field of a generic record created contains 1 random value"
+ */
+public class GenericRecordFullPayloadSizeEstimator implements Serializable {
+
+ private final transient Schema baseSchema;
+
+ // This variable is used to track the number of complex/collection fields with primitive data types at their leaf.
+ // This is used to figure out how many entries can be packed in such a collection field to meet the min payload
+ // size requested
+ private final transient AtomicInteger counter = new AtomicInteger(0);
+
+ public GenericRecordFullPayloadSizeEstimator(Schema schema) {
+ this.baseSchema = schema;
+ }
+
+ public Pair typeEstimateAndNumComplexFields() {
+ int size = estimate(baseSchema);
+ return Pair.of(size, counter.get());
+ }
+
+ /**
+ * This method estimates the size of the payload if all entries of this payload were populated with one value.
+ * For eg. A primitive data type such as String will be populated with {@link UUID} so the length if 36 bytes
+ * whereas a complex data type such as an Array of type Int, will be populated with exactly 1 Integer value.
+ */
+ protected int estimate(Schema schema) {
+ long size = 0;
+ for (Schema.Field f : schema.getFields()) {
+ size += typeEstimate(f.schema());
+ }
+ return (int) size;
+ }
+
+ private long typeEstimate(Schema schema) {
+ Schema localSchema = schema;
+ if (isOption(schema)) {
+ localSchema = getNonNull(schema);
+ }
+ switch (localSchema.getType()) {
+ case BOOLEAN:
+ return 1;
+ case DOUBLE:
+ return 8;
+ case FLOAT:
+ return 4;
+ case INT:
+ return 4;
+ case LONG:
+ return 8;
+ case STRING:
+ return UUID.randomUUID().toString().length();
+ case ENUM:
+ return 1;
+ case RECORD:
+ return estimate(localSchema);
+ case ARRAY:
+ if (GenericRecordFullPayloadGenerator.isPrimitive(localSchema.getElementType())) {
+ counter.addAndGet(1);
+ }
+ Schema elementSchema = localSchema.getElementType();
+ return typeEstimate(elementSchema);
+ case MAP:
+ if (GenericRecordFullPayloadGenerator.isPrimitive(localSchema.getValueType())) {
+ counter.addAndGet(1);
+ }
+ Schema valueSchema = localSchema.getValueType();
+ return UUID.randomUUID().toString().length() + typeEstimate(valueSchema);
+ case BYTES:
+ return UUID.randomUUID().toString().length();
+ case FIXED:
+ return localSchema.getFixedSize();
+ default:
+ throw new IllegalArgumentException(
+ "Cannot handle type: " + localSchema.getType());
+ }
+ }
+
+ protected boolean isOption(Schema schema) {
+ return schema.getType().equals(Schema.Type.UNION)
+ && schema.getTypes().size() == 2
+ && (schema.getTypes().get(0).getType().equals(Schema.Type.NULL)
+ || schema.getTypes().get(1).getType().equals(Schema.Type.NULL));
+ }
+
+ protected Schema getNonNull(Schema schema) {
+ List types = schema.getTypes();
+ return types.get(0).getType().equals(Schema.Type.NULL) ? types.get(1) : types.get(0);
+ }
+
+}
+
diff --git a/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/generator/GenericRecordPartialPayloadGenerator.java b/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/generator/GenericRecordPartialPayloadGenerator.java
new file mode 100644
index 000000000..f7e4174e6
--- /dev/null
+++ b/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/generator/GenericRecordPartialPayloadGenerator.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.integ.testsuite.generator;
+
+import org.apache.avro.Schema;
+import org.apache.avro.generic.GenericData;
+import org.apache.avro.generic.GenericRecord;
+
+/**
+ * This is a GenericRecord payload generator that generates partial generic records {@link GenericRecord}. A partial
+ * records is one that has some fields of the schema NULL or NOT PRESENT. This payload enables us to simulate
+ * creation of partial records which are possible in many cases, especially for database change logs.
+ */
+public class GenericRecordPartialPayloadGenerator extends GenericRecordFullPayloadGenerator {
+
+ public GenericRecordPartialPayloadGenerator(Schema schema) {
+ super(schema);
+ }
+
+ public GenericRecordPartialPayloadGenerator(Schema schema, int minPayloadSize) {
+ super(schema, minPayloadSize);
+ }
+
+ @Override
+ protected GenericRecord convert(Schema schema) {
+ GenericRecord record = super.convertPartial(schema);
+ return record;
+ }
+
+ private void setNull(GenericRecord record) {
+ for (Schema.Field field : record.getSchema().getFields()) {
+ // A random boolean decides whether this field of the generic record should be present or absent.
+ // Using this we can set only a handful of fields in the record and generate partial records
+ boolean setNull = random.nextBoolean();
+ if (setNull) { // TODO : DO NOT SET THE RECORD KEY FIELDS TO NULL
+ record.put(field.name(), null);
+ } else {
+ if (record.get(field.name()) instanceof GenericData.Record) {
+ setNull((GenericData.Record) record.get(field.name()));
+ }
+ }
+ }
+ }
+
+ @Override
+ public boolean validate(GenericRecord record) {
+ return validate((Object) record);
+ }
+
+ // Atleast 1 entry should be null
+ private boolean validate(Object object) {
+ if (object == null) {
+ return true;
+ } else if (object instanceof GenericRecord) {
+ for (Schema.Field field : ((GenericRecord) object).getSchema().getFields()) {
+ boolean ret = validate(((GenericRecord) object).get(field.name()));
+ if (ret) {
+ return ret;
+ }
+ }
+ }
+ return false;
+ }
+
+}
+
diff --git a/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/generator/LazyRecordGeneratorIterator.java b/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/generator/LazyRecordGeneratorIterator.java
new file mode 100644
index 000000000..a775d37c5
--- /dev/null
+++ b/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/generator/LazyRecordGeneratorIterator.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.integ.testsuite.generator;
+
+import java.util.Iterator;
+import org.apache.avro.generic.GenericRecord;
+import org.apache.hudi.client.utils.LazyIterableIterator;
+
+/**
+ * A lazy record generator to generate {@link GenericRecord}s lazily and not hold a list of records in memory.
+ */
+public class LazyRecordGeneratorIterator extends
+ LazyIterableIterator {
+
+ public LazyRecordGeneratorIterator(Iterator inputItr) {
+ super(inputItr);
+ }
+
+ @Override
+ protected void start() {
+ }
+
+ @Override
+ protected GenericRecord computeNext() {
+ return inputItr.next();
+ }
+
+ @Override
+ protected void end() {
+
+ }
+}
diff --git a/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/generator/UpdateGeneratorIterator.java b/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/generator/UpdateGeneratorIterator.java
new file mode 100644
index 000000000..685a5c77a
--- /dev/null
+++ b/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/generator/UpdateGeneratorIterator.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.integ.testsuite.generator;
+
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import org.apache.avro.Schema;
+import org.apache.avro.generic.GenericRecord;
+
+public class UpdateGeneratorIterator implements Iterator {
+
+ // Use the full payload generator as default
+ private GenericRecordFullPayloadGenerator generator;
+ private List blackListedFields;
+ // iterator
+ private Iterator itr;
+
+ public UpdateGeneratorIterator(Iterator itr, String schemaStr, List partitionPathFieldNames,
+ List recordKeyFieldNames, int minPayloadSize) {
+ this.itr = itr;
+ this.blackListedFields = new ArrayList<>();
+ this.blackListedFields.addAll(partitionPathFieldNames);
+ this.blackListedFields.addAll(recordKeyFieldNames);
+ Schema schema = new Schema.Parser().parse(schemaStr);
+ this.generator = new GenericRecordFullPayloadGenerator(schema, minPayloadSize);
+ }
+
+ @Override
+ public boolean hasNext() {
+ return itr.hasNext();
+ }
+
+ @Override
+ public GenericRecord next() {
+ GenericRecord newRecord = itr.next();
+ return this.generator.randomize(newRecord, this.blackListedFields);
+ }
+
+}
diff --git a/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/helpers/DFSTestSuitePathSelector.java b/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/helpers/DFSTestSuitePathSelector.java
new file mode 100644
index 000000000..b67e21fe2
--- /dev/null
+++ b/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/helpers/DFSTestSuitePathSelector.java
@@ -0,0 +1,92 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.integ.testsuite.helpers;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.stream.Collectors;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.LocatedFileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.RemoteIterator;
+import org.apache.hudi.common.config.TypedProperties;
+import org.apache.hudi.common.util.Option;
+import org.apache.hudi.common.util.collection.ImmutablePair;
+import org.apache.hudi.common.util.collection.Pair;
+import org.apache.hudi.exception.HoodieIOException;
+import org.apache.hudi.utilities.sources.helpers.DFSPathSelector;
+
+/**
+ * A custom dfs path selector used only for the hudi test suite. To be used only if workload is not run inline.
+ */
+public class DFSTestSuitePathSelector extends DFSPathSelector {
+
+ public DFSTestSuitePathSelector(TypedProperties props, Configuration hadoopConf) {
+ super(props, hadoopConf);
+ }
+
+ @Override
+ public Pair, String> getNextFilePathsAndMaxModificationTime(
+ Option lastCheckpointStr, long sourceLimit) {
+
+ Integer lastBatchId;
+ Integer nextBatchId;
+ try {
+ if (lastCheckpointStr.isPresent()) {
+ lastBatchId = Integer.parseInt(lastCheckpointStr.get());
+ nextBatchId = lastBatchId + 1;
+ } else {
+ lastBatchId = -1;
+ nextBatchId = 0;
+ }
+ // obtain all eligible files for the batch
+ List eligibleFiles = new ArrayList<>();
+ FileStatus[] fileStatuses = fs.globStatus(
+ new Path(props.getString(Config.ROOT_INPUT_PATH_PROP), "*"));
+ for (FileStatus fileStatus : fileStatuses) {
+ if (!fileStatus.isDirectory() || IGNORE_FILEPREFIX_LIST.stream()
+ .anyMatch(pfx -> fileStatus.getPath().getName().startsWith(pfx))) {
+ continue;
+ } else if (fileStatus.getPath().getName().compareTo(lastBatchId.toString()) > 0 && fileStatus.getPath()
+ .getName().compareTo(nextBatchId.toString()) <= 0) {
+ RemoteIterator files = fs.listFiles(fileStatus.getPath(), true);
+ while (files.hasNext()) {
+ eligibleFiles.add(files.next());
+ }
+ }
+ }
+ // no data to readAvro
+ if (eligibleFiles.size() == 0) {
+ return new ImmutablePair<>(Option.empty(),
+ lastCheckpointStr.orElseGet(() -> String.valueOf(Long.MIN_VALUE)));
+ }
+ // readAvro the files out.
+ String pathStr = eligibleFiles.stream().map(f -> f.getPath().toString())
+ .collect(Collectors.joining(","));
+
+ return new ImmutablePair<>(Option.ofNullable(pathStr), String.valueOf(nextBatchId));
+ } catch (IOException ioe) {
+ throw new HoodieIOException(
+ "Unable to readAvro from source from checkpoint: " + lastCheckpointStr, ioe);
+ }
+ }
+
+}
diff --git a/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/helpers/HiveServiceProvider.java b/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/helpers/HiveServiceProvider.java
new file mode 100644
index 000000000..88022ed1c
--- /dev/null
+++ b/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/helpers/HiveServiceProvider.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.integ.testsuite.helpers;
+
+import java.io.IOException;
+import java.net.BindException;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hive.service.server.HiveServer2;
+import org.apache.hudi.hive.testutils.HiveTestService;
+import org.apache.hudi.integ.testsuite.HoodieTestSuiteWriter;
+import org.apache.hudi.integ.testsuite.configuration.DeltaConfig.Config;
+import org.apache.log4j.LogManager;
+import org.apache.log4j.Logger;
+import org.apache.thrift.transport.TTransportException;
+
+/**
+ * Hive Service provider.
+ */
+public class HiveServiceProvider {
+
+ private HiveTestService hiveService;
+ private HiveServer2 hiveServer;
+ private Config config;
+
+ private static final Logger LOG = LogManager.getLogger(HiveServiceProvider.class);
+
+ public HiveServiceProvider(Config config) {
+ this.config = config;
+ }
+
+ public void startLocalHiveServiceIfNeeded(Configuration configuration) throws IOException {
+ if (config.isHiveLocal()) {
+ hiveService = new HiveTestService(configuration);
+ hiveServer = hiveService.start();
+ }
+ }
+
+ public void syncToLocalHiveIfNeeded(HoodieTestSuiteWriter writer) {
+ if (this.config.isHiveLocal()) {
+ writer.getDeltaStreamerWrapper().getDeltaSyncService().getDeltaSync()
+ .syncHive(getLocalHiveServer().getHiveConf());
+ } else {
+ writer.getDeltaStreamerWrapper().getDeltaSyncService().getDeltaSync().syncHive();
+ }
+ }
+
+ public void stopLocalHiveServiceIfNeeded() throws IOException {
+ if (config.isHiveLocal()) {
+ if (hiveService != null) {
+ hiveService.stop();
+ }
+ }
+ }
+
+ public HiveServer2 getLocalHiveServer() {
+ return hiveServer;
+ }
+}
diff --git a/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/reader/DFSAvroDeltaInputReader.java b/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/reader/DFSAvroDeltaInputReader.java
new file mode 100644
index 000000000..f1bb02a69
--- /dev/null
+++ b/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/reader/DFSAvroDeltaInputReader.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.integ.testsuite.reader;
+
+import java.io.IOException;
+import java.util.Arrays;
+import org.apache.avro.generic.GenericRecord;
+import org.apache.hadoop.fs.PathFilter;
+import org.apache.hudi.common.util.Option;
+import org.apache.hudi.integ.testsuite.writer.AvroFileDeltaInputWriter;
+import org.apache.hudi.integ.testsuite.writer.DeltaOutputMode;
+import org.apache.spark.api.java.JavaRDD;
+import org.apache.spark.sql.SparkSession;
+
+/**
+ * A reader of {@link DeltaOutputMode#DFS} and {@link DeltaInputType#AVRO}.
+ */
+public class DFSAvroDeltaInputReader extends DFSDeltaInputReader {
+
+ private final SparkSession sparkSession;
+ private final String schemaStr;
+ private final String basePath;
+ private final Option structName;
+ private final Option nameSpace;
+ protected PathFilter filter = (path) -> {
+ if (path.toUri().toString().contains(AvroFileDeltaInputWriter.AVRO_EXTENSION)) {
+ return true;
+ } else {
+ return false;
+ }
+ };
+
+ public DFSAvroDeltaInputReader(SparkSession sparkSession, String schemaStr, String basePath,
+ Option structName,
+ Option nameSpace) {
+ this.sparkSession = sparkSession;
+ this.schemaStr = schemaStr;
+ this.basePath = basePath;
+ this.structName = structName;
+ this.nameSpace = nameSpace;
+ }
+
+ @Override
+ public JavaRDD read(long totalRecordsToRead) throws IOException {
+ return SparkBasedReader.readAvro(sparkSession, schemaStr, getFilePathsToRead(basePath, filter, totalRecordsToRead),
+ structName, nameSpace);
+ }
+
+ @Override
+ public JavaRDD read(int numPartitions, long approxNumRecords) throws IOException {
+ throw new UnsupportedOperationException("cannot generate updates");
+ }
+
+ @Override
+ public JavaRDD read(int numPartitions, int numFiles, long approxNumRecords) throws IOException {
+ throw new UnsupportedOperationException("cannot generate updates");
+ }
+
+ @Override
+ public JavaRDD read(int numPartitions, int numFiles, double percentageRecordsPerFile)
+ throws IOException {
+ throw new UnsupportedOperationException("cannot generate updates");
+ }
+
+ @Override
+ protected long analyzeSingleFile(String filePath) {
+ JavaRDD recordsFromOneFile = SparkBasedReader
+ .readAvro(sparkSession, schemaStr, Arrays.asList(filePath),
+ structName, nameSpace);
+ return recordsFromOneFile.count();
+ }
+
+}
diff --git a/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/reader/DFSDeltaInputReader.java b/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/reader/DFSDeltaInputReader.java
new file mode 100644
index 000000000..da9268161
--- /dev/null
+++ b/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/reader/DFSDeltaInputReader.java
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.integ.testsuite.reader;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import org.apache.avro.generic.GenericRecord;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.PathFilter;
+import org.apache.hudi.common.fs.FSUtils;
+import org.apache.hudi.common.util.collection.Pair;
+
+/**
+ * This class helps to estimate the number of files to read a given number of total records.
+ * Use this class for all DFS based implementations of {@link DeltaInputReader}
+ */
+public abstract class DFSDeltaInputReader implements DeltaInputReader {
+
+ protected List getFilePathsToRead(String basePath, PathFilter filter, long totalRecordsToRead) throws
+ IOException {
+ FileSystem fs = FSUtils.getFs(basePath, new Configuration());
+ // TODO : Sort list by file size and take the median file status to ensure fair calculation and change to remote
+ // iterator
+ List fileStatuses = Arrays.asList(fs.globStatus(new Path(basePath, "*/*"), filter));
+ if (fileStatuses.size() > 0) {
+ FileStatus status = fileStatuses.get(0);
+ long avgNumRecordsPerFile = analyzeSingleFile(status.getPath().toString());
+ long numFilesToMatchExpectedRecords = (long) Math.ceil((double) totalRecordsToRead / (double)
+ avgNumRecordsPerFile);
+ long avgSizeOfEachFile = status.getLen();
+ long totalSizeToRead = avgSizeOfEachFile * numFilesToMatchExpectedRecords;
+ // choose N files with that length
+ Pair fileStatusIndexRange = getFileStatusIndexRange(fileStatuses, avgSizeOfEachFile,
+ totalSizeToRead);
+ int startIndex = fileStatusIndexRange.getLeft();
+ List filePaths = new ArrayList<>();
+ while (startIndex == 0 || startIndex < fileStatusIndexRange.getRight()) {
+ filePaths.add(fileStatuses.get(startIndex).getPath().toString());
+ startIndex++;
+ }
+ return filePaths;
+ }
+ return Collections.emptyList();
+ }
+
+ protected Pair getFileStatusIndexRange(List fileStatuses, long averageFileSize, long
+ totalSizeToRead) {
+ long totalSizeOfFilesPresent = 0;
+ int startOffset = 0;
+ int endOffset = 0;
+ for (FileStatus fileStatus : fileStatuses) {
+ // If current file length is greater than averageFileSize, increment by averageFileSize since our
+ // totalSizeToRead calculation is based on the averageRecordSize * numRecordsToRead.
+ if (fileStatus.getLen() > averageFileSize) {
+ totalSizeOfFilesPresent += averageFileSize;
+ } else {
+ totalSizeOfFilesPresent += fileStatus.getLen();
+ }
+ if (totalSizeOfFilesPresent <= totalSizeToRead) {
+ endOffset++;
+ continue;
+ } else {
+ return Pair.of(startOffset, endOffset);
+ }
+ }
+ return Pair.of(startOffset, endOffset);
+ }
+
+ /**
+ * Implementation of {@link DeltaInputReader}s to provide a way to read a single file on DFS and provide an
+ * average number of records across N files.
+ */
+ protected long analyzeSingleFile(String filePath) {
+ throw new UnsupportedOperationException("No implementation found");
+ }
+}
diff --git a/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/reader/DFSHoodieDatasetInputReader.java b/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/reader/DFSHoodieDatasetInputReader.java
new file mode 100644
index 000000000..209aa469c
--- /dev/null
+++ b/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/reader/DFSHoodieDatasetInputReader.java
@@ -0,0 +1,338 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.integ.testsuite.reader;
+
+import static java.util.Map.Entry.comparingByValue;
+import static java.util.stream.Collectors.toMap;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.NoSuchElementException;
+import java.util.stream.Collectors;
+import java.util.stream.StreamSupport;
+import org.apache.avro.Schema;
+import org.apache.avro.generic.GenericRecord;
+import org.apache.avro.generic.IndexedRecord;
+import org.apache.hadoop.fs.Path;
+import org.apache.hudi.avro.HoodieAvroUtils;
+import org.apache.hudi.common.fs.FSUtils;
+import org.apache.hudi.common.model.FileSlice;
+import org.apache.hudi.common.model.HoodieRecord;
+import org.apache.hudi.common.model.HoodieRecordPayload;
+import org.apache.hudi.common.table.HoodieTableMetaClient;
+import org.apache.hudi.common.table.log.HoodieMergedLogRecordScanner;
+import org.apache.hudi.common.table.view.HoodieTableFileSystemView;
+import org.apache.hudi.common.table.view.TableFileSystemView;
+import org.apache.hudi.common.util.Option;
+import org.apache.hudi.common.util.ParquetReaderIterator;
+import org.apache.hudi.common.util.ValidationUtils;
+import org.apache.hudi.config.HoodieMemoryConfig;
+import org.apache.parquet.avro.AvroParquetReader;
+import org.apache.spark.api.java.JavaPairRDD;
+import org.apache.spark.api.java.JavaRDD;
+import org.apache.spark.api.java.JavaSparkContext;
+import org.apache.spark.sql.SparkSession;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import scala.Tuple2;
+
+/**
+ * This class helps to generate updates from an already existing hoodie dataset. It supports generating updates in
+ * across partitions, files and records.
+ */
+public class DFSHoodieDatasetInputReader extends DFSDeltaInputReader {
+
+ private static Logger log = LoggerFactory.getLogger(DFSHoodieDatasetInputReader.class);
+
+ private transient JavaSparkContext jsc;
+ private String schemaStr;
+ private HoodieTableMetaClient metaClient;
+
+ public DFSHoodieDatasetInputReader(JavaSparkContext jsc, String basePath, String schemaStr) {
+ this.jsc = jsc;
+ this.schemaStr = schemaStr;
+ this.metaClient = new HoodieTableMetaClient(jsc.hadoopConfiguration(), basePath);
+ }
+
+ protected List getPartitions(Option partitionsLimit) throws IOException {
+ List partitionPaths = FSUtils
+ .getAllPartitionPaths(metaClient.getFs(), metaClient.getBasePath(), false);
+ // Sort partition so we can pick last N partitions by default
+ Collections.sort(partitionPaths);
+ if (!partitionPaths.isEmpty()) {
+ ValidationUtils.checkArgument(partitionPaths.size() >= partitionsLimit.get(),
+ "Cannot generate updates for more partitions " + "than present in the dataset, partitions "
+ + "requested " + partitionsLimit.get() + ", partitions present " + partitionPaths.size());
+ return partitionPaths.subList(0, partitionsLimit.get());
+ }
+ return partitionPaths;
+
+ }
+
+ private JavaPairRDD> getPartitionToFileSlice(HoodieTableMetaClient metaClient,
+ List partitionPaths) {
+ TableFileSystemView.SliceView fileSystemView = new HoodieTableFileSystemView(metaClient,
+ metaClient.getCommitsAndCompactionTimeline().filterCompletedInstants());
+ // pass num partitions to another method
+ JavaPairRDD> partitionToFileSliceList = jsc.parallelize(partitionPaths).mapToPair(p -> {
+ return new Tuple2<>(p, fileSystemView.getLatestFileSlices(p).iterator());
+ });
+ return partitionToFileSliceList;
+ }
+
+ @Override
+ protected long analyzeSingleFile(String filePath) {
+ return SparkBasedReader.readParquet(new SparkSession(jsc.sc()), Arrays.asList(filePath),
+ Option.empty(), Option.empty()).count();
+ }
+
+ private JavaRDD fetchAnyRecordsFromDataset(Option numRecordsToUpdate) throws IOException {
+ return fetchRecordsFromDataset(Option.empty(), Option.empty(), numRecordsToUpdate, Option.empty());
+ }
+
+ private JavaRDD fetchAnyRecordsFromDataset(Option numRecordsToUpdate, Option
+ numPartitions) throws IOException {
+ return fetchRecordsFromDataset(numPartitions, Option.empty(), numRecordsToUpdate, Option.empty());
+ }
+
+ private JavaRDD fetchPercentageRecordsFromDataset(Option numPartitions, Option
+ numFiles, Option percentageRecordsPerFile) throws IOException {
+ return fetchRecordsFromDataset(numPartitions, numFiles, Option.empty(), percentageRecordsPerFile);
+ }
+
+ private JavaRDD fetchRecordsFromDataset(Option numPartitions, Option
+ numFiles, Option numRecordsToUpdate) throws IOException {
+ return fetchRecordsFromDataset(numPartitions, numFiles, numRecordsToUpdate, Option.empty());
+ }
+
+ private JavaRDD fetchRecordsFromDataset(Option numPartitions, Option numFiles,
+ Option numRecordsToUpdate, Option percentageRecordsPerFile) throws IOException {
+ log.info("NumPartitions : {}, NumFiles : {}, numRecordsToUpdate : {}, percentageRecordsPerFile : {}",
+ numPartitions, numFiles, numRecordsToUpdate, percentageRecordsPerFile);
+ List partitionPaths = getPartitions(numPartitions);
+ // Read all file slices in the partition
+ JavaPairRDD> partitionToFileSlice = getPartitionToFileSlice(metaClient,
+ partitionPaths);
+ // TODO : read record count from metadata
+ // Read the records in a single file
+ long recordsInSingleFile = iteratorSize(readParquetOrLogFiles(getSingleSliceFromRDD(partitionToFileSlice)));
+ int numFilesToUpdate;
+ long numRecordsToUpdatePerFile;
+ if (!numFiles.isPresent() || numFiles.get() == 0) {
+ // If num files are not passed, find the number of files to update based on total records to update and records
+ // per file
+ numFilesToUpdate = (int) (numRecordsToUpdate.get() / recordsInSingleFile);
+ log.info("Files to update {}", numFilesToUpdate);
+ numRecordsToUpdatePerFile = recordsInSingleFile;
+ } else {
+ // If num files is passed, find the number of records per file based on either percentage or total records to
+ // update and num files passed
+ numFilesToUpdate = numFiles.get();
+ numRecordsToUpdatePerFile = percentageRecordsPerFile.isPresent() ? (long) (recordsInSingleFile
+ * percentageRecordsPerFile.get()) : numRecordsToUpdate.get() / numFilesToUpdate;
+ }
+ // Adjust the number of files to read per partition based on the requested partition & file counts
+ Map adjustedPartitionToFileIdCountMap = getFilesToReadPerPartition(partitionToFileSlice,
+ getPartitions(numPartitions).size(), numFilesToUpdate);
+ JavaRDD updates = projectSchema(generateUpdates(adjustedPartitionToFileIdCountMap,
+ partitionToFileSlice, numFilesToUpdate, (int) numRecordsToUpdatePerFile));
+ if (numRecordsToUpdate.isPresent() && numFiles.isPresent() && numFiles.get() != 0 && numRecordsToUpdate.get()
+ != numRecordsToUpdatePerFile * numFiles.get()) {
+ long remainingRecordsToAdd = (numRecordsToUpdate.get() - (numRecordsToUpdatePerFile * numFiles.get()));
+ updates = updates.union(projectSchema(jsc.parallelize(generateUpdates(adjustedPartitionToFileIdCountMap,
+ partitionToFileSlice, numFilesToUpdate, (int) remainingRecordsToAdd).take((int) remainingRecordsToAdd))));
+ }
+ log.info("Finished generating updates");
+ return updates;
+ }
+
+ private JavaRDD projectSchema(JavaRDD updates) {
+ // The records read from the hoodie dataset have the hoodie record fields, rewrite the record to eliminate them
+ return updates
+ .map(r -> HoodieAvroUtils.rewriteRecordWithOnlyNewSchemaFields(r, new Schema.Parser().parse(schemaStr)));
+ }
+
+ private JavaRDD generateUpdates(Map adjustedPartitionToFileIdCountMap,
+ JavaPairRDD> partitionToFileSlice, int numFiles, int numRecordsToReadPerFile) {
+ return partitionToFileSlice.map(p -> {
+ int maxFilesToRead = adjustedPartitionToFileIdCountMap.get(p._1);
+ return iteratorLimit(p._2, maxFilesToRead);
+ }).flatMap(p -> p).repartition(numFiles).map(fileSlice -> {
+ if (numRecordsToReadPerFile > 0) {
+ return iteratorLimit(readParquetOrLogFiles(fileSlice), numRecordsToReadPerFile);
+ } else {
+ return readParquetOrLogFiles(fileSlice);
+ }
+ }).flatMap(p -> p).map(i -> (GenericRecord) i);
+ }
+
+ private Map getFilesToReadPerPartition(JavaPairRDD>
+ partitionToFileSlice, Integer numPartitions, Integer numFiles) {
+ int numFilesPerPartition = (int) Math.ceil(numFiles / numPartitions);
+ Map partitionToFileIdCountMap = partitionToFileSlice
+ .mapToPair(p -> new Tuple2<>(p._1, iteratorSize(p._2))).collectAsMap();
+ long totalExistingFilesCount = partitionToFileIdCountMap.values().stream().reduce((a, b) -> a + b).get();
+ ValidationUtils.checkArgument(totalExistingFilesCount >= numFiles, "Cannot generate updates "
+ + "for more files than present in the dataset, file requested " + numFiles + ", files present "
+ + totalExistingFilesCount);
+ Map partitionToFileIdCountSortedMap = partitionToFileIdCountMap
+ .entrySet()
+ .stream()
+ .sorted(comparingByValue())
+ .collect(toMap(Map.Entry::getKey, Map.Entry::getValue, (e1, e2) -> e2,
+ LinkedHashMap::new));
+ // Limit files to be read per partition
+ Map adjustedPartitionToFileIdCountMap = new HashMap<>();
+ partitionToFileIdCountSortedMap.entrySet().stream().forEach(e -> {
+ if (e.getValue() <= numFilesPerPartition) {
+ adjustedPartitionToFileIdCountMap.put(e.getKey(), e.getValue());
+ } else {
+ adjustedPartitionToFileIdCountMap.put(e.getKey(), numFilesPerPartition);
+ }
+ });
+ return adjustedPartitionToFileIdCountMap;
+ }
+
+ private FileSlice getSingleSliceFromRDD(JavaPairRDD> partitionToFileSlice) {
+ return partitionToFileSlice.map(f -> {
+ FileSlice slice = f._2.next();
+ FileSlice newSlice = new FileSlice(slice.getFileGroupId(), slice.getBaseInstantTime());
+ if (slice.getBaseFile().isPresent()) {
+ newSlice.setBaseFile(slice.getBaseFile().get());
+ } else {
+ slice.getLogFiles().forEach(l -> {
+ newSlice.addLogFile(l);
+ });
+ }
+ return newSlice;
+ }).take(1).get(0);
+ }
+
+ private Iterator readParquetOrLogFiles(FileSlice fileSlice) throws IOException {
+ if (fileSlice.getBaseFile().isPresent()) {
+ Iterator itr =
+ new ParquetReaderIterator(AvroParquetReader.builder(new
+ Path(fileSlice.getBaseFile().get().getPath())).withConf(metaClient.getHadoopConf()).build());
+ return itr;
+ } else {
+ // If there is no data file, fall back to reading log files
+ HoodieMergedLogRecordScanner scanner = new HoodieMergedLogRecordScanner(metaClient.getFs(),
+ metaClient.getBasePath(),
+ fileSlice.getLogFiles().map(l -> l.getPath().getName()).collect(Collectors.toList()),
+ new Schema.Parser().parse(schemaStr), metaClient.getActiveTimeline().getCommitsTimeline()
+ .filterCompletedInstants().lastInstant().get().getTimestamp(),
+ HoodieMemoryConfig.DEFAULT_MAX_MEMORY_FOR_SPILLABLE_MAP_IN_BYTES, true, false,
+ HoodieMemoryConfig.DEFAULT_MAX_DFS_STREAM_BUFFER_SIZE,
+ HoodieMemoryConfig.DEFAULT_SPILLABLE_MAP_BASE_PATH);
+ // readAvro log files
+ Iterable> iterable = () -> scanner.iterator();
+ Schema schema = new Schema.Parser().parse(schemaStr);
+ return StreamSupport.stream(iterable.spliterator(), false)
+ .map(e -> {
+ try {
+ return (IndexedRecord) e.getData().getInsertValue(schema).get();
+ } catch (IOException io) {
+ throw new UncheckedIOException(io);
+ }
+ }).iterator();
+ }
+ }
+
+ /**
+ * Returns the number of elements remaining in {@code iterator}. The iterator
+ * will be left exhausted: its {@code hasNext()} method will return
+ * {@code false}.
+ */
+ private static int iteratorSize(Iterator iterator) {
+ int count = 0;
+ while (iterator.hasNext()) {
+ iterator.next();
+ count++;
+ }
+ return count;
+ }
+
+ /**
+ * Creates an iterator returning the first {@code limitSize} elements of the
+ * given iterator. If the original iterator does not contain that many
+ * elements, the returned iterator will have the same behavior as the original
+ * iterator. The returned iterator supports {@code remove()} if the original
+ * iterator does.
+ *
+ * @param iterator the iterator to limit
+ * @param limitSize the maximum number of elements in the returned iterator
+ * @throws IllegalArgumentException if {@code limitSize} is negative
+ */
+ private static Iterator iteratorLimit(
+ final Iterator iterator, final int limitSize) {
+ ValidationUtils.checkArgument(iterator != null, "iterator is null");
+ ValidationUtils.checkArgument(limitSize >= 0, "limit is negative");
+ return new Iterator() {
+ private int count;
+
+ @Override
+ public boolean hasNext() {
+ return count < limitSize && iterator.hasNext();
+ }
+
+ @Override
+ public T next() {
+ if (!hasNext()) {
+ throw new NoSuchElementException();
+ }
+ count++;
+ return iterator.next();
+ }
+
+ @Override
+ public void remove() {
+ iterator.remove();
+ }
+ };
+ }
+
+ @Override
+ public JavaRDD read(long numRecords) throws IOException {
+ return fetchAnyRecordsFromDataset(Option.of(numRecords));
+ }
+
+ @Override
+ public JavaRDD read(int numPartitions, long approxNumRecords) throws IOException {
+ return fetchAnyRecordsFromDataset(Option.of(approxNumRecords), Option.of(numPartitions));
+ }
+
+ @Override
+ public JavaRDD read(int numPartitions, int numFiles, long numRecords) throws IOException {
+ return fetchRecordsFromDataset(Option.of(numPartitions), Option.of(numFiles), Option.of(numRecords));
+ }
+
+ @Override
+ public JavaRDD read(int numPartitions, int numFiles, double percentageRecordsPerFile)
+ throws IOException {
+ return fetchPercentageRecordsFromDataset(Option.of(numPartitions), Option.of(numFiles),
+ Option.of(percentageRecordsPerFile));
+ }
+}
diff --git a/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/reader/DeltaInputReader.java b/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/reader/DeltaInputReader.java
new file mode 100644
index 000000000..3ba404153
--- /dev/null
+++ b/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/reader/DeltaInputReader.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.integ.testsuite.reader;
+
+import java.io.IOException;
+import java.io.Serializable;
+import org.apache.spark.api.java.JavaRDD;
+
+/**
+ * Implementations of {@link DeltaInputReader} will read the configured input type and provide an RDD of records to the
+ * client.
+ *
+ * @param Read result data type
+ */
+public interface DeltaInputReader extends Serializable {
+
+ /**
+ * Attempts to reads an approximate number of records close to approxNumRecords.
+ * This highly depends on the number of records already present in the input.
+ */
+ JavaRDD read(long approxNumRecords) throws IOException;
+
+ /**
+ * @throws IOException Attempts to read approx number of records (exact if equal or more records available)
+ * across requested number of
+ * partitions.
+ */
+ JavaRDD read(int numPartitions, long approxNumRecords) throws IOException;
+
+ /**
+ * @throws IOException Attempts to read approx number of records (exact if equal or more records available)
+ * across requested number of
+ * partitions and number of files.
+ * 1. Find numFiles across numPartitions
+ * 2. numRecordsToReadPerFile = approxNumRecords / numFiles
+ */
+ JavaRDD read(int numPartitions, int numFiles, long approxNumRecords) throws IOException;
+
+ /**
+ * @throws IOException Attempts to a % of records per file across requested number of partitions and number of files.
+ * 1. Find numFiles across numPartitions
+ * 2. numRecordsToReadPerFile = approxNumRecordsPerFile * percentageRecordsPerFile
+ */
+ JavaRDD read(int numPartitions, int numFiles, double percentageRecordsPerFile) throws IOException;
+
+}
diff --git a/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/reader/DeltaInputType.java b/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/reader/DeltaInputType.java
new file mode 100644
index 000000000..137a503ee
--- /dev/null
+++ b/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/reader/DeltaInputType.java
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.integ.testsuite.reader;
+
+/**
+ * Supported delta input data types.
+ */
+public enum DeltaInputType {
+ AVRO, PARQUET
+}
diff --git a/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/reader/SparkBasedReader.java b/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/reader/SparkBasedReader.java
new file mode 100644
index 000000000..16a5259ee
--- /dev/null
+++ b/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/reader/SparkBasedReader.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.integ.testsuite.reader;
+
+import java.util.List;
+import org.apache.avro.generic.GenericRecord;
+import org.apache.hudi.AvroConversionUtils;
+import org.apache.hudi.common.util.Option;
+import org.apache.hudi.utilities.schema.RowBasedSchemaProvider;
+import org.apache.spark.api.java.JavaRDD;
+import org.apache.spark.sql.Dataset;
+import org.apache.spark.sql.Row;
+import org.apache.spark.sql.SparkSession;
+import scala.collection.JavaConverters;
+
+
+/**
+ * Helper class to read avro and/or parquet files and generate a RDD of {@link GenericRecord}.
+ */
+public class SparkBasedReader {
+
+ public static final String SPARK_AVRO_FORMAT = "avro";
+ public static final String SPARK_PARQUET_FORMAT = "com.databricks.spark.parquet";
+ private static final String AVRO_SCHEMA_OPTION_KEY = "avroSchema";
+
+ // Spark anyways globs the path and gets all the paths in memory so take the List