[HUDI-242] Support for RFC-12/Bootstrapping of external datasets to hudi (#1876)

- [HUDI-418] Bootstrap Index Implementation using HFile with unit-test
 - [HUDI-421] FileSystem View Changes to support Bootstrap with unit-tests
 - [HUDI-424] Implement Query Side Integration for querying tables containing bootstrap file slices
 - [HUDI-423] Implement upsert functionality for handling updates to these bootstrap file slices
 - [HUDI-421] Bootstrap Write Client with tests
 - [HUDI-425] Added HoodieDeltaStreamer support
 - [HUDI-899] Add a knob to change partition-path style while performing metadata bootstrap
 - [HUDI-900] Metadata Bootstrap Key Generator needs to handle complex keys correctly
 - [HUDI-424] Simplify Record reader implementation
 - [HUDI-423] Implement upsert functionality for handling updates to these bootstrap file slices
 - [HUDI-420] Hoodie Demo working with hive and sparkSQL. Also, Hoodie CLI working with bootstrap tables

Co-authored-by: Mehrotra <uditme@amazon.com>
Co-authored-by: Vinoth Chandar <vinoth@apache.org>
Co-authored-by: Balaji Varadarajan <varadarb@uber.com>

hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/HoodieDeltaStreamerWrapper.java

@@ -23,6 +23,7 @@ import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hudi.client.WriteStatus;
 import org.apache.hudi.common.model.HoodieRecord;
 import org.apache.hudi.common.util.collection.Pair;
+import org.apache.hudi.utilities.deltastreamer.DeltaSync;
 import org.apache.hudi.utilities.deltastreamer.HoodieDeltaStreamer;
 import org.apache.hudi.utilities.schema.SchemaProvider;
 import org.apache.spark.api.java.JavaRDD;
@@ -39,14 +40,9 @@ public class HoodieDeltaStreamerWrapper extends HoodieDeltaStreamer {
     super(cfg, jssc);
   }
 
-  public HoodieDeltaStreamerWrapper(Config cfg, JavaSparkContext jssc, FileSystem fs, HiveConf conf) throws Exception {
-    super(cfg, jssc, fs, conf);
-  }
-
-  public JavaRDD<WriteStatus> upsert(Operation operation) throws
-      Exception {
+  public JavaRDD<WriteStatus> upsert(Operation operation) throws Exception {
     cfg.operation = operation;
-    return deltaSyncService.getDeltaSync().syncOnce().getRight();
+    return deltaSyncService.get().getDeltaSync().syncOnce().getRight();
   }
 
   public JavaRDD<WriteStatus> insert() throws Exception {
@@ -71,7 +67,8 @@ public class HoodieDeltaStreamerWrapper extends HoodieDeltaStreamer {
   }
 
   public Pair<SchemaProvider, Pair<String, JavaRDD<HoodieRecord>>> fetchSource() throws Exception {
-    return deltaSyncService.getDeltaSync().readFromSource(deltaSyncService.getDeltaSync().getCommitTimelineOpt());
+    DeltaSync service = deltaSyncService.get().getDeltaSync();
+    return service.readFromSource(service.getCommitTimelineOpt());
   }
 
 }

hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/HoodieTestSuiteJob.java

@@ -41,7 +41,7 @@ import org.apache.hudi.integ.testsuite.dag.scheduler.DagScheduler;
 import org.apache.hudi.integ.testsuite.generator.DeltaGenerator;
 import org.apache.hudi.integ.testsuite.reader.DeltaInputType;
 import org.apache.hudi.integ.testsuite.writer.DeltaOutputMode;
-import org.apache.hudi.keygen.KeyGenerator;
+import org.apache.hudi.keygen.BuiltinKeyGenerator;
 import org.apache.hudi.utilities.UtilHelpers;
 import org.apache.hudi.utilities.deltastreamer.HoodieDeltaStreamer;
 import org.apache.hudi.utilities.schema.SchemaProvider;
@@ -85,7 +85,7 @@ public class HoodieTestSuiteJob {
    */
   private transient HiveConf hiveConf;
 
-  private KeyGenerator keyGenerator;
+  private BuiltinKeyGenerator keyGenerator;
 
   public HoodieTestSuiteJob(HoodieTestSuiteConfig cfg, JavaSparkContext jsc) throws IOException {
     this.cfg = cfg;
@@ -96,7 +96,8 @@ public class HoodieTestSuiteJob {
     log.info("Creating workload generator with configs : {}", props.toString());
     this.schemaProvider = UtilHelpers.createSchemaProvider(cfg.schemaProviderClassName, props, jsc);
     this.hiveConf = getDefaultHiveConf(jsc.hadoopConfiguration());
-    this.keyGenerator = DataSourceUtils.createKeyGenerator(props);
+    this.keyGenerator = (BuiltinKeyGenerator) DataSourceUtils.createKeyGenerator(props);
+
     if (!fs.exists(new Path(cfg.targetBasePath))) {
       HoodieTableMetaClient.initTableType(jsc.hadoopConfiguration(), cfg.targetBasePath,
           HoodieTableType.valueOf(cfg.tableType), cfg.targetTableName, "archived");

hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/generator/DeltaGenerator.java

@@ -40,6 +40,7 @@ import org.apache.hudi.integ.testsuite.writer.DeltaOutputMode;
 import org.apache.hudi.integ.testsuite.writer.DeltaWriteStats;
 import org.apache.hudi.integ.testsuite.writer.DeltaWriterAdapter;
 import org.apache.hudi.integ.testsuite.writer.DeltaWriterFactory;
+import org.apache.hudi.keygen.BuiltinKeyGenerator;
 import org.apache.hudi.keygen.ComplexKeyGenerator;
 import org.apache.hudi.keygen.KeyGenerator;
 import org.apache.hudi.keygen.SimpleKeyGenerator;
@@ -70,16 +71,13 @@ public class DeltaGenerator implements Serializable {
   private int batchId;
 
   public DeltaGenerator(DeltaConfig deltaOutputConfig, JavaSparkContext jsc, SparkSession sparkSession,
-      String schemaStr,
-      KeyGenerator keyGenerator) {
+                        String schemaStr, BuiltinKeyGenerator keyGenerator) {
     this.deltaOutputConfig = deltaOutputConfig;
     this.jsc = jsc;
     this.sparkSession = sparkSession;
     this.schemaStr = schemaStr;
-    this.recordRowKeyFieldNames = keyGenerator instanceof ComplexKeyGenerator ? ((ComplexKeyGenerator) keyGenerator)
-        .getRecordKeyFields() : Arrays.asList(((SimpleKeyGenerator) keyGenerator).getRecordKeyField());
-    this.partitionPathFieldNames = keyGenerator instanceof ComplexKeyGenerator ? ((ComplexKeyGenerator) keyGenerator)
-        .getPartitionPathFields() : Arrays.asList(((SimpleKeyGenerator) keyGenerator).getPartitionPathField());
+    this.recordRowKeyFieldNames = keyGenerator.getRecordKeyFields();
+    this.partitionPathFieldNames = keyGenerator.getPartitionPathFields();
   }
 
   public JavaRDD<DeltaWriteStats> writeRecords(JavaRDD<GenericRecord> records) {

hudi-integ-test/src/test/java/org/apache/hudi/integ/ITTestBase.java

@@ -162,7 +162,9 @@ public abstract class ITTestBase {
         .awaitCompletion();
     int exitCode = dockerClient.inspectExecCmd(createCmdResponse.getId()).exec().getExitCode();
     LOG.info("Exit code for command : " + exitCode);
-    LOG.error("\n\n ###### Stdout #######\n" + callback.getStdout().toString());
+    if (exitCode != 0) {
+      LOG.error("\n\n ###### Stdout #######\n" + callback.getStdout().toString());
+    }
     LOG.error("\n\n ###### Stderr #######\n" + callback.getStderr().toString());
 
     if (expectedToSucceed) {
@@ -240,6 +242,9 @@ public abstract class ITTestBase {
       String filePath = System.getProperty("java.io.tmpdir") + "/" + System.currentTimeMillis() + "-hive.log";
       FileIOUtils.writeStringToFile(hiveLogStr, filePath);
       LOG.info("Hive log saved up at  : " + filePath);
+      LOG.info("<===========  Full hive log ===============>\n"
+          + "\n" + hiveLogStr
+          + "\n <==========================================>");
     } catch (Exception e) {
       LOG.error("Unable to save up logs..", e);
     }
@@ -268,7 +273,7 @@ public abstract class ITTestBase {
       saveUpLogs();
     }
 
-    assertEquals(times, count, "Did not find output the expected number of times");
+    assertEquals(times, count, "Did not find output the expected number of times.");
   }
 
   public class TestExecStartResultCallback extends ExecStartResultCallback {

hudi-integ-test/src/test/java/org/apache/hudi/integ/ITTestHoodieDemo.java

@@ -22,6 +22,7 @@ import org.apache.hudi.common.model.HoodieFileFormat;
 import org.apache.hudi.common.util.CollectionUtils;
 import org.apache.hudi.common.util.collection.Pair;
 
+import org.apache.hudi.keygen.SimpleKeyGenerator;
 import org.junit.jupiter.api.Test;
 
 import java.util.Collections;
@@ -53,11 +54,18 @@ public class ITTestHoodieDemo extends ITTestBase {
   private static final String COW_TABLE_NAME = "stock_ticks_cow";
   private static final String MOR_TABLE_NAME = "stock_ticks_mor";
 
+  private static final String BOOTSTRAPPED_SRC_PATH = "/user/hive/warehouse/stock_ticks_cow_bs_src";
+  private static final String COW_BOOTSTRAPPED_BASE_PATH = "/user/hive/warehouse/stock_ticks_cow_bs";
+  private static final String MOR_BOOTSTRAPPED_BASE_PATH = "/user/hive/warehouse/stock_ticks_mor_bs";
+  private static final String COW_BOOTSTRAPPED_TABLE_NAME = "stock_ticks_cow_bs";
+  private static final String MOR_BOOTSTRAPPED_TABLE_NAME = "stock_ticks_mor_bs";
+
   private static final String DEMO_CONTAINER_SCRIPT = HOODIE_WS_ROOT + "/docker/demo/setup_demo_container.sh";
   private static final String MIN_COMMIT_TIME_COW_SCRIPT = HOODIE_WS_ROOT + "/docker/demo/get_min_commit_time_cow.sh";
   private static final String MIN_COMMIT_TIME_MOR_SCRIPT = HOODIE_WS_ROOT + "/docker/demo/get_min_commit_time_mor.sh";
   private static final String HUDI_CLI_TOOL = HOODIE_WS_ROOT + "/hudi-cli/hudi-cli.sh";
   private static final String COMPACTION_COMMANDS = HOODIE_WS_ROOT + "/docker/demo/compaction.commands";
+  private static final String SPARKSQL_BS_PREP_COMMANDS = HOODIE_WS_ROOT + "/docker/demo/sparksql-bootstrap-prep-source.commands";
   private static final String SPARKSQL_BATCH1_COMMANDS = HOODIE_WS_ROOT + "/docker/demo/sparksql-batch1.commands";
   private static final String SPARKSQL_BATCH2_COMMANDS = HOODIE_WS_ROOT + "/docker/demo/sparksql-batch2.commands";
   private static final String SPARKSQL_INCREMENTAL_COMMANDS = HOODIE_WS_ROOT + "/docker/demo/sparksql-incremental.commands";
@@ -100,6 +108,7 @@ public class ITTestHoodieDemo extends ITTestBase {
 
     // compaction
     scheduleAndRunCompaction();
+
     testHiveAfterSecondBatchAfterCompaction();
     testPrestoAfterSecondBatchAfterCompaction();
     testIncrementalHiveQueryAfterCompaction();
@@ -155,40 +164,80 @@ public class ITTestHoodieDemo extends ITTestBase {
             + " --disable-compaction " + String.format(HIVE_SYNC_CMD_FMT, "dt", MOR_TABLE_NAME)));
 
     executeCommandStringsInDocker(ADHOC_1_CONTAINER, cmds);
+    executeSparkSQLCommand(SPARKSQL_BS_PREP_COMMANDS, true);
+    List<String> bootstrapCmds = CollectionUtils.createImmutableList(
+        "spark-submit --class org.apache.hudi.utilities.deltastreamer.HoodieDeltaStreamer " + HUDI_UTILITIES_BUNDLE
+        + " --table-type COPY_ON_WRITE "
+        + " --run-bootstrap "
+        + " --source-class org.apache.hudi.utilities.sources.JsonDFSSource --source-ordering-field ts "
+        + " --target-base-path " + COW_BOOTSTRAPPED_BASE_PATH + " --target-table " + COW_BOOTSTRAPPED_TABLE_NAME
+        + " --props /var/demo/config/dfs-source.properties"
+        + " --schemaprovider-class org.apache.hudi.utilities.schema.FilebasedSchemaProvider "
+        + " --initial-checkpoint-provider"
+        + " org.apache.hudi.utilities.checkpointing.InitialCheckpointFromAnotherHoodieTimelineProvider"
+        + " --hoodie-conf hoodie.bootstrap.base.path=" + BOOTSTRAPPED_SRC_PATH
+        + " --hoodie-conf hoodie.bootstrap.recordkey.columns=key"
+        + " --hoodie-conf hoodie.deltastreamer.checkpoint.provider.path=" + COW_BASE_PATH
+        + " --hoodie-conf hoodie.bootstrap.parallelism=2 "
+        + " --hoodie-conf hoodie.bootstrap.keygen.class=" + SimpleKeyGenerator.class.getName()
+        + String.format(HIVE_SYNC_CMD_FMT, "dt", COW_BOOTSTRAPPED_TABLE_NAME),
+        "spark-submit --class org.apache.hudi.utilities.deltastreamer.HoodieDeltaStreamer " + HUDI_UTILITIES_BUNDLE
+        + " --table-type MERGE_ON_READ "
+        + " --run-bootstrap "
+        + " --source-class org.apache.hudi.utilities.sources.JsonDFSSource --source-ordering-field ts "
+        + " --target-base-path " + MOR_BOOTSTRAPPED_BASE_PATH + " --target-table " + MOR_BOOTSTRAPPED_TABLE_NAME
+        + " --props /var/demo/config/dfs-source.properties"
+        + " --schemaprovider-class org.apache.hudi.utilities.schema.FilebasedSchemaProvider "
+        + " --initial-checkpoint-provider"
+        + " org.apache.hudi.utilities.checkpointing.InitialCheckpointFromAnotherHoodieTimelineProvider"
+        + " --hoodie-conf hoodie.bootstrap.base.path=" + BOOTSTRAPPED_SRC_PATH
+        + " --hoodie-conf hoodie.bootstrap.recordkey.columns=key"
+        + " --hoodie-conf hoodie.deltastreamer.checkpoint.provider.path=" + COW_BASE_PATH
+        + " --hoodie-conf hoodie.bootstrap.parallelism=2 "
+        + " --hoodie-conf hoodie.bootstrap.keygen.class=" + SimpleKeyGenerator.class.getName()
+        + String.format(HIVE_SYNC_CMD_FMT, "dt", MOR_BOOTSTRAPPED_TABLE_NAME));
+    executeCommandStringsInDocker(ADHOC_1_CONTAINER, bootstrapCmds);
   }
 
   private void testHiveAfterFirstBatch() throws Exception {
     Pair<String, String> stdOutErrPair = executeHiveCommandFile(HIVE_TBLCHECK_COMMANDS);
     assertStdOutContains(stdOutErrPair, "| stock_ticks_cow     |");
+    assertStdOutContains(stdOutErrPair, "| stock_ticks_cow_bs  |");
     assertStdOutContains(stdOutErrPair, "| stock_ticks_mor_ro  |");
     assertStdOutContains(stdOutErrPair, "| stock_ticks_mor_rt  |");
-
+    assertStdOutContains(stdOutErrPair, "| stock_ticks_mor_bs_ro  |");
+    assertStdOutContains(stdOutErrPair, "| stock_ticks_mor_bs_rt  |");
     assertStdOutContains(stdOutErrPair,
         "|   partition    |\n+----------------+\n| dt=2018-08-31  |\n+----------------+\n", 3);
 
     stdOutErrPair = executeHiveCommandFile(HIVE_BATCH1_COMMANDS);
     assertStdOutContains(stdOutErrPair, "| symbol  |         _c1          |\n+---------+----------------------+\n"
-        + "| GOOG    | 2018-08-31 10:29:00  |\n", 3);
+        + "| GOOG    | 2018-08-31 10:29:00  |\n", 6);
     assertStdOutContains(stdOutErrPair,
         "| symbol  |          ts          | volume  |    open    |   close   |\n"
             + "+---------+----------------------+---------+------------+-----------+\n"
             + "| GOOG    | 2018-08-31 09:59:00  | 6330    | 1230.5     | 1230.02   |\n"
             + "| GOOG    | 2018-08-31 10:29:00  | 3391    | 1230.1899  | 1230.085  |\n",
-        3);
+        6);
   }
 
   private void testSparkSQLAfterFirstBatch() throws Exception {
     Pair<String, String> stdOutErrPair = executeSparkSQLCommand(SPARKSQL_BATCH1_COMMANDS, true);
     assertStdOutContains(stdOutErrPair, "|default |stock_ticks_cow   |false      |\n"
+                                                    + "|default |stock_ticks_cow_bs   |false      |\n"
+                                                    + "|default |stock_ticks_mor_bs_ro |false      |\n"
+                                                    +  "|default |stock_ticks_mor_bs_rt |false      |"
                                                     + "|default |stock_ticks_mor_ro |false      |\n"
-                                                    + "|default |stock_ticks_mor_rt |false      |");
+                                                    +  "|default |stock_ticks_mor_rt |false      |");
     assertStdOutContains(stdOutErrPair,
-        "+------+-------------------+\n|GOOG  |2018-08-31 10:29:00|\n+------+-------------------+", 3);
-    assertStdOutContains(stdOutErrPair, "|GOOG  |2018-08-31 09:59:00|6330  |1230.5   |1230.02 |", 3);
-    assertStdOutContains(stdOutErrPair, "|GOOG  |2018-08-31 10:29:00|3391  |1230.1899|1230.085|", 3);
+        "+------+-------------------+\n|GOOG  |2018-08-31 10:29:00|\n+------+-------------------+", 6);
+    assertStdOutContains(stdOutErrPair, "|GOOG  |2018-08-31 09:59:00|6330  |1230.5   |1230.02 |", 6);
+    assertStdOutContains(stdOutErrPair, "|GOOG  |2018-08-31 10:29:00|3391  |1230.1899|1230.085|", 6);
   }
 
   private void ingestSecondBatchAndHiveSync() throws Exception {
+    // Note : Unlike normal tables, bootstrapped tables do not have checkpoint. So, they
+    // begin with null checkpoint and read all states.
     List<String> cmds = CollectionUtils.createImmutableList(
             ("hdfs dfs -copyFromLocal -f " + INPUT_BATCH_PATH2 + " " + HDFS_BATCH_PATH2),
             ("spark-submit"
@@ -208,14 +257,28 @@ public class ITTestHoodieDemo extends ITTestBase {
             + " --target-base-path " + MOR_BASE_PATH + " --target-table " + MOR_TABLE_NAME
             + " --props /var/demo/config/dfs-source.properties"
             + " --schemaprovider-class org.apache.hudi.utilities.schema.FilebasedSchemaProvider "
-            + " --disable-compaction " + String.format(HIVE_SYNC_CMD_FMT, "dt", MOR_TABLE_NAME)));
+            + " --disable-compaction " + String.format(HIVE_SYNC_CMD_FMT, "dt", MOR_TABLE_NAME)),
+            ("spark-submit --class org.apache.hudi.utilities.deltastreamer.HoodieDeltaStreamer " + HUDI_UTILITIES_BUNDLE
+            + " --table-type COPY_ON_WRITE "
+            + " --source-class org.apache.hudi.utilities.sources.JsonDFSSource --source-ordering-field ts "
+            + " --target-base-path " + COW_BOOTSTRAPPED_BASE_PATH + " --target-table " + COW_BOOTSTRAPPED_TABLE_NAME
+            + " --props /var/demo/config/dfs-source.properties"
+            + " --schemaprovider-class org.apache.hudi.utilities.schema.FilebasedSchemaProvider "
+            + String.format(HIVE_SYNC_CMD_FMT, "dt", COW_BOOTSTRAPPED_TABLE_NAME)),
+            ("spark-submit --class org.apache.hudi.utilities.deltastreamer.HoodieDeltaStreamer " + HUDI_UTILITIES_BUNDLE
+            + " --table-type MERGE_ON_READ "
+            + " --source-class org.apache.hudi.utilities.sources.JsonDFSSource --source-ordering-field ts "
+            + " --target-base-path " + MOR_BOOTSTRAPPED_BASE_PATH + " --target-table " + MOR_BOOTSTRAPPED_TABLE_NAME
+            + " --props /var/demo/config/dfs-source.properties"
+            + " --schemaprovider-class org.apache.hudi.utilities.schema.FilebasedSchemaProvider "
+            + " --disable-compaction " + String.format(HIVE_SYNC_CMD_FMT, "dt", MOR_BOOTSTRAPPED_TABLE_NAME)));
     executeCommandStringsInDocker(ADHOC_1_CONTAINER, cmds);
   }
 
   private void testPrestoAfterFirstBatch() throws Exception {
     Pair<String, String> stdOutErrPair = executePrestoCommandFile(HDFS_PRESTO_INPUT_TABLE_CHECK_PATH);
-    assertStdOutContains(stdOutErrPair, "stock_ticks_cow");
-    assertStdOutContains(stdOutErrPair, "stock_ticks_mor",2);
+    assertStdOutContains(stdOutErrPair, "stock_ticks_cow", 2);
+    assertStdOutContains(stdOutErrPair, "stock_ticks_mor",4);
 
     stdOutErrPair = executePrestoCommandFile(HDFS_PRESTO_INPUT_BATCH1_PATH);
     assertStdOutContains(stdOutErrPair,
@@ -229,20 +292,20 @@ public class ITTestHoodieDemo extends ITTestBase {
   private void testHiveAfterSecondBatch() throws Exception {
     Pair<String, String> stdOutErrPair = executeHiveCommandFile(HIVE_BATCH1_COMMANDS);
     assertStdOutContains(stdOutErrPair, "| symbol  |         _c1          |\n+---------+----------------------+\n"
-        + "| GOOG    | 2018-08-31 10:29:00  |\n");
+        + "| GOOG    | 2018-08-31 10:29:00  |\n", 2);
     assertStdOutContains(stdOutErrPair, "| symbol  |         _c1          |\n+---------+----------------------+\n"
-        + "| GOOG    | 2018-08-31 10:59:00  |\n", 2);
+        + "| GOOG    | 2018-08-31 10:59:00  |\n", 4);
     assertStdOutContains(stdOutErrPair,
         "| symbol  |          ts          | volume  |    open    |   close   |\n"
             + "+---------+----------------------+---------+------------+-----------+\n"
             + "| GOOG    | 2018-08-31 09:59:00  | 6330    | 1230.5     | 1230.02   |\n"
-            + "| GOOG    | 2018-08-31 10:29:00  | 3391    | 1230.1899  | 1230.085  |\n");
+            + "| GOOG    | 2018-08-31 10:29:00  | 3391    | 1230.1899  | 1230.085  |\n", 2);
     assertStdOutContains(stdOutErrPair,
         "| symbol  |          ts          | volume  |    open    |   close   |\n"
             + "+---------+----------------------+---------+------------+-----------+\n"
             + "| GOOG    | 2018-08-31 09:59:00  | 6330    | 1230.5     | 1230.02   |\n"
             + "| GOOG    | 2018-08-31 10:59:00  | 9021    | 1227.1993  | 1227.215  |\n",
-        2);
+        4);
   }
 
   private void testPrestoAfterSecondBatch() throws Exception {
@@ -262,13 +325,13 @@ public class ITTestHoodieDemo extends ITTestBase {
   private void testHiveAfterSecondBatchAfterCompaction() throws Exception {
     Pair<String, String> stdOutErrPair = executeHiveCommandFile(HIVE_BATCH2_COMMANDS);
     assertStdOutContains(stdOutErrPair, "| symbol  |         _c1          |\n+---------+----------------------+\n"
-        + "| GOOG    | 2018-08-31 10:59:00  |", 2);
+        + "| GOOG    | 2018-08-31 10:59:00  |", 4);
     assertStdOutContains(stdOutErrPair,
         "| symbol  |          ts          | volume  |    open    |   close   |\n"
             + "+---------+----------------------+---------+------------+-----------+\n"
             + "| GOOG    | 2018-08-31 09:59:00  | 6330    | 1230.5     | 1230.02   |\n"
             + "| GOOG    | 2018-08-31 10:59:00  | 9021    | 1227.1993  | 1227.215  |",
-        2);
+        4);
   }
 
   private void testPrestoAfterSecondBatchAfterCompaction() throws Exception {
@@ -284,13 +347,13 @@ public class ITTestHoodieDemo extends ITTestBase {
   private void testSparkSQLAfterSecondBatch() throws Exception {
     Pair<String, String> stdOutErrPair = executeSparkSQLCommand(SPARKSQL_BATCH2_COMMANDS, true);
     assertStdOutContains(stdOutErrPair,
-        "+------+-------------------+\n|GOOG  |2018-08-31 10:59:00|\n+------+-------------------+", 2);
+        "+------+-------------------+\n|GOOG  |2018-08-31 10:59:00|\n+------+-------------------+", 4);
 
-    assertStdOutContains(stdOutErrPair, "|GOOG  |2018-08-31 09:59:00|6330  |1230.5   |1230.02 |", 3);
-    assertStdOutContains(stdOutErrPair, "|GOOG  |2018-08-31 10:59:00|9021  |1227.1993|1227.215|", 2);
+    assertStdOutContains(stdOutErrPair, "|GOOG  |2018-08-31 09:59:00|6330  |1230.5   |1230.02 |", 6);
+    assertStdOutContains(stdOutErrPair, "|GOOG  |2018-08-31 10:59:00|9021  |1227.1993|1227.215|", 4);
     assertStdOutContains(stdOutErrPair,
-        "+------+-------------------+\n|GOOG  |2018-08-31 10:29:00|\n+------+-------------------+");
-    assertStdOutContains(stdOutErrPair, "|GOOG  |2018-08-31 10:29:00|3391  |1230.1899|1230.085|");
+        "+------+-------------------+\n|GOOG  |2018-08-31 10:29:00|\n+------+-------------------+", 2);
+    assertStdOutContains(stdOutErrPair, "|GOOG  |2018-08-31 10:29:00|3391  |1230.1899|1230.085|", 2);
   }
 
   private void testIncrementalHiveQuery(String minCommitTimeScript, String incrementalCommandsFile,
@@ -306,36 +369,40 @@ public class ITTestHoodieDemo extends ITTestBase {
     String expectedOutput = "| GOOG    | 2018-08-31 10:59:00  | 9021    | 1227.1993  | 1227.215  |";
 
     // verify that 10:59 is present in COW table because there is no compaction process for COW
-    testIncrementalHiveQuery(MIN_COMMIT_TIME_COW_SCRIPT, HIVE_INCREMENTAL_COW_COMMANDS, expectedOutput, 1);
+    testIncrementalHiveQuery(MIN_COMMIT_TIME_COW_SCRIPT, HIVE_INCREMENTAL_COW_COMMANDS, expectedOutput, 2);
 
     // verify that 10:59 is NOT present in RO table because of pending compaction
     testIncrementalHiveQuery(MIN_COMMIT_TIME_MOR_SCRIPT, HIVE_INCREMENTAL_MOR_RO_COMMANDS, expectedOutput, 0);
 
     // verify that 10:59 is present in RT table even with pending compaction
-    testIncrementalHiveQuery(MIN_COMMIT_TIME_MOR_SCRIPT, HIVE_INCREMENTAL_MOR_RT_COMMANDS, expectedOutput, 1);
+    testIncrementalHiveQuery(MIN_COMMIT_TIME_MOR_SCRIPT, HIVE_INCREMENTAL_MOR_RT_COMMANDS, expectedOutput, 2);
   }
 
   private void testIncrementalHiveQueryAfterCompaction() throws Exception {
     String expectedOutput = "| symbol  |          ts          | volume  |    open    |   close   |\n"
-            + "+---------+----------------------+---------+------------+-----------+\n"
-            + "| GOOG    | 2018-08-31 10:59:00  | 9021    | 1227.1993  | 1227.215  |";
+        + "+---------+----------------------+---------+------------+-----------+\n"
+        + "| GOOG    | 2018-08-31 10:59:00  | 9021    | 1227.1993  | 1227.215  |";
 
     // verify that 10:59 is present for all views because compaction is complete
-    testIncrementalHiveQuery(MIN_COMMIT_TIME_COW_SCRIPT, HIVE_INCREMENTAL_COW_COMMANDS, expectedOutput, 1);
-    testIncrementalHiveQuery(MIN_COMMIT_TIME_MOR_SCRIPT, HIVE_INCREMENTAL_MOR_RO_COMMANDS, expectedOutput, 1);
-    testIncrementalHiveQuery(MIN_COMMIT_TIME_MOR_SCRIPT, HIVE_INCREMENTAL_MOR_RT_COMMANDS, expectedOutput, 1);
+    testIncrementalHiveQuery(MIN_COMMIT_TIME_COW_SCRIPT, HIVE_INCREMENTAL_COW_COMMANDS, expectedOutput, 2);
+    testIncrementalHiveQuery(MIN_COMMIT_TIME_MOR_SCRIPT, HIVE_INCREMENTAL_MOR_RO_COMMANDS, expectedOutput, 2);
+    testIncrementalHiveQuery(MIN_COMMIT_TIME_MOR_SCRIPT, HIVE_INCREMENTAL_MOR_RT_COMMANDS, expectedOutput, 2);
   }
 
   private void testIncrementalSparkSQLQuery() throws Exception {
     Pair<String, String> stdOutErrPair = executeSparkSQLCommand(SPARKSQL_INCREMENTAL_COMMANDS, true);
-    assertStdOutContains(stdOutErrPair, "|GOOG  |2018-08-31 10:59:00|9021  |1227.1993|1227.215|");
-    assertStdOutContains(stdOutErrPair, "|default |stock_ticks_cow           |false      |\n"
-        + "|default |stock_ticks_derived_mor_ro|false      |\n"
-        + "|default |stock_ticks_derived_mor_rt|false      |\n"
-        + "|default |stock_ticks_mor_ro        |false      |\n"
-        + "|default |stock_ticks_mor_rt        |false      |\n"
-        + "|        |stock_ticks_cow_incr      |true       |");
-    assertStdOutContains(stdOutErrPair, "|count(1)|\n+--------+\n|99     |", 2);
+    assertStdOutContains(stdOutErrPair, "|GOOG  |2018-08-31 10:59:00|9021  |1227.1993|1227.215|", 2);
+    assertStdOutContains(stdOutErrPair, "|default |stock_ticks_cow              |false      |\n"
+        + "|default |stock_ticks_cow_bs           |false      |\n"
+        + "|default |stock_ticks_derived_mor_bs_ro|false      |\n"
+        + "|default |stock_ticks_derived_mor_bs_rt|false      |\n"
+        + "|default |stock_ticks_derived_mor_ro   |false      |\n"
+        + "|default |stock_ticks_derived_mor_rt   |false      |\n"
+        + "|default |stock_ticks_mor_bs_ro        |false      |\n"
+        + "|default |stock_ticks_mor_bs_rt        |false      |"
+        + "|default |stock_ticks_mor_ro           |false      |\n"
+        + "|default |stock_ticks_mor_rt           |false      |");
+    assertStdOutContains(stdOutErrPair, "|count(1)|\n+--------+\n|99     |", 4);
   }
 
   private void scheduleAndRunCompaction() throws Exception {