Adding ability for inserts to be written to log files

hoodie-common/src/main/java/com/uber/hoodie/common/table/HoodieTableMetaClient.java

@@ -16,12 +16,15 @@
 
 package com.uber.hoodie.common.table;
 
+import static com.uber.hoodie.common.model.HoodieTableType.MERGE_ON_READ;
+
 import com.uber.hoodie.common.SerializableConfiguration;
 import com.uber.hoodie.common.model.HoodieTableType;
 import com.uber.hoodie.common.table.timeline.HoodieActiveTimeline;
 import com.uber.hoodie.common.table.timeline.HoodieArchivedTimeline;
 import com.uber.hoodie.common.util.FSUtils;
 import com.uber.hoodie.exception.DatasetNotFoundException;
+import com.uber.hoodie.exception.HoodieException;
 import java.io.File;
 import java.io.IOException;
 import java.io.Serializable;
@@ -198,7 +201,7 @@ public class HoodieTableMetaClient implements Serializable {
     Properties properties = new Properties();
     properties.setProperty(HoodieTableConfig.HOODIE_TABLE_NAME_PROP_NAME, tableName);
     properties.setProperty(HoodieTableConfig.HOODIE_TABLE_TYPE_PROP_NAME, tableType.name());
-    if (tableType == HoodieTableType.MERGE_ON_READ) {
+    if (tableType == MERGE_ON_READ) {
       properties.setProperty(HoodieTableConfig.HOODIE_PAYLOAD_CLASS_PROP_NAME, payloadClassName);
     }
     return HoodieTableMetaClient.initializePathAsHoodieDataset(hadoopConf, basePath, properties);
@@ -254,6 +257,53 @@ public class HoodieTableMetaClient implements Serializable {
     return fs.listStatus(metaPath, nameFilter);
   }
 
+  /**
+   * Get the commit timeline visible for this table
+   */
+  public HoodieTimeline getCommitsTimeline() {
+    switch (this.getTableType()) {
+      case COPY_ON_WRITE:
+        return getActiveTimeline().getCommitTimeline();
+      case MERGE_ON_READ:
+        // We need to include the parquet files written out in delta commits
+        // Include commit action to be able to start doing a MOR over a COW dataset - no
+        // migration required
+        return getActiveTimeline().getCommitsTimeline();
+      default:
+        throw new HoodieException("Unsupported table type :" + this.getTableType());
+    }
+  }
+
+  /**
+   * Get the compacted commit timeline visible for this table
+   */
+  public HoodieTimeline getCommitTimeline() {
+    switch (this.getTableType()) {
+      case COPY_ON_WRITE:
+      case MERGE_ON_READ:
+        // We need to include the parquet files written out in delta commits in tagging
+        return getActiveTimeline().getCommitTimeline();
+      default:
+        throw new HoodieException("Unsupported table type :" + this.getTableType());
+    }
+  }
+
+  /**
+   * Gets the commit action type
+   */
+  public String getCommitActionType() {
+    switch (this.getTableType()) {
+      case COPY_ON_WRITE:
+        return HoodieActiveTimeline.COMMIT_ACTION;
+      case MERGE_ON_READ:
+        return HoodieActiveTimeline.DELTA_COMMIT_ACTION;
+      default:
+        throw new HoodieException(
+            "Could not commit on unknown storage type " + this.getTableType());
+    }
+  }
+
+
   @Override
   public boolean equals(Object o) {
     if (this == o) {

hoodie-common/src/main/java/com/uber/hoodie/common/table/log/AbstractHoodieLogRecordScanner.java

@@ -91,13 +91,12 @@ public abstract class AbstractHoodieLogRecordScanner {
   private AtomicLong totalCorruptBlocks = new AtomicLong(0);
   // Store the last instant log blocks (needed to implement rollback)
   private Deque<HoodieLogBlock> currentInstantLogBlocks = new ArrayDeque<>();
-
   // Progress
   private float progress = 0.0f;
 
-  public AbstractHoodieLogRecordScanner(FileSystem fs, String basePath, List<String> logFilePaths,
-      Schema readerSchema, String latestInstantTime,
-      boolean readBlocksLazily, boolean reverseReader, int bufferSize) {
+  // TODO (NA) - Change this to a builder, this constructor is too long
+  public AbstractHoodieLogRecordScanner(FileSystem fs, String basePath, List<String> logFilePaths, Schema readerSchema,
+      String latestInstantTime, boolean readBlocksLazily, boolean reverseReader, int bufferSize) {
     this.readerSchema = readerSchema;
     this.latestInstantTime = latestInstantTime;
     this.hoodieTableMetaClient = new HoodieTableMetaClient(fs.getConf(), basePath);

hoodie-common/src/main/java/com/uber/hoodie/common/util/FSUtils.java

@@ -247,6 +247,13 @@ public class FSUtils {
     return LOG_FILE_PREFIX + String.format("%s_%s%s*", fileId, commitTime, logFileExtension);
   }
 
+  public static boolean isLogFile(Path logPath) {
+    Matcher matcher = LOG_FILE_PATTERN.matcher(logPath.getName());
+    if (!matcher.find()) {
+      return false;
+    }
+    return true;
+  }
 
   /**
    * Get the latest log file written from the list of log files passed in