HUDI-138 - Meta Files handling also need to support consistency guard

hoodie-cli/src/main/java/com/uber/hoodie/cli/HoodieCLI.java

@@ -19,6 +19,7 @@
 package com.uber.hoodie.cli;
 
 import com.uber.hoodie.common.table.HoodieTableMetaClient;
+import com.uber.hoodie.common.util.ConsistencyGuardConfig;
 import com.uber.hoodie.common.util.FSUtils;
 import java.io.IOException;
 import org.apache.hadoop.conf.Configuration;
@@ -27,8 +28,10 @@ import org.apache.hadoop.fs.FileSystem;
 public class HoodieCLI {
 
   public static Configuration conf;
+  public static ConsistencyGuardConfig consistencyGuardConfig = ConsistencyGuardConfig.newBuilder().build();
   public static FileSystem fs;
   public static CLIState state = CLIState.INIT;
+  public static String basePath;
   public static HoodieTableMetaClient tableMetadata;
   public static HoodieTableMetaClient syncTableMetadata;
 
@@ -37,6 +40,18 @@ public class HoodieCLI {
     INIT, DATASET, SYNC
   }
 
+  public static void setConsistencyGuardConfig(ConsistencyGuardConfig config) {
+    consistencyGuardConfig = config;
+  }
+
+  private static void setTableMetaClient(HoodieTableMetaClient tableMetadata) {
+    HoodieCLI.tableMetadata = tableMetadata;
+  }
+
+  private static void setBasePath(String basePath) {
+    HoodieCLI.basePath = basePath;
+  }
+
   public static boolean initConf() {
     if (HoodieCLI.conf == null) {
       HoodieCLI.conf = FSUtils.prepareHadoopConf(new Configuration());
@@ -47,11 +62,16 @@ public class HoodieCLI {
 
   public static void initFS(boolean force) throws IOException {
     if (fs == null || force) {
-      fs = FileSystem.get(conf);
+      fs = (tableMetadata != null) ? tableMetadata.getFs() : FileSystem.get(conf);
     }
   }
 
-  public static void setTableMetadata(HoodieTableMetaClient tableMetadata) {
-    HoodieCLI.tableMetadata = tableMetadata;
+  public static void refreshTableMetadata() {
+    setTableMetaClient(new HoodieTableMetaClient(HoodieCLI.conf, basePath, false, HoodieCLI.consistencyGuardConfig));
+  }
+
+  public static void connectTo(String basePath) {
+    setBasePath(basePath);
+    refreshTableMetadata();
   }
 }

hoodie-cli/src/main/java/com/uber/hoodie/cli/commands/CleansCommand.java

@@ -23,7 +23,6 @@ import com.uber.hoodie.avro.model.HoodieCleanPartitionMetadata;
 import com.uber.hoodie.cli.HoodieCLI;
 import com.uber.hoodie.cli.HoodiePrintHelper;
 import com.uber.hoodie.cli.TableHeader;
-import com.uber.hoodie.common.table.HoodieTableMetaClient;
 import com.uber.hoodie.common.table.HoodieTimeline;
 import com.uber.hoodie.common.table.timeline.HoodieActiveTimeline;
 import com.uber.hoodie.common.table.timeline.HoodieInstant;
@@ -91,9 +90,8 @@ public class CleansCommand implements CommandMarker {
 
   @CliCommand(value = "cleans refresh", help = "Refresh the commits")
   public String refreshCleans() throws IOException {
-    HoodieTableMetaClient metadata = new HoodieTableMetaClient(HoodieCLI.conf, HoodieCLI.tableMetadata.getBasePath());
-    HoodieCLI.setTableMetadata(metadata);
-    return "Metadata for table " + metadata.getTableConfig().getTableName() + " refreshed.";
+    HoodieCLI.refreshTableMetadata();
+    return "Metadata for table " + HoodieCLI.tableMetadata.getTableConfig().getTableName() + " refreshed.";
   }
 
   @CliCommand(value = "clean showpartitions", help = "Show partition level details of a clean")

hoodie-cli/src/main/java/com/uber/hoodie/cli/commands/CommitsCommand.java

@@ -115,9 +115,8 @@ public class CommitsCommand implements CommandMarker {
 
   @CliCommand(value = "commits refresh", help = "Refresh the commits")
   public String refreshCommits() throws IOException {
-    HoodieTableMetaClient metadata = new HoodieTableMetaClient(HoodieCLI.conf, HoodieCLI.tableMetadata.getBasePath());
-    HoodieCLI.setTableMetadata(metadata);
-    return "Metadata for table " + metadata.getTableConfig().getTableName() + " refreshed.";
+    HoodieCLI.refreshTableMetadata();
+    return "Metadata for table " + HoodieCLI.tableMetadata.getTableConfig().getTableName() + " refreshed.";
   }
 
   @CliCommand(value = "commit rollback", help = "Rollback a commit")

hoodie-cli/src/main/java/com/uber/hoodie/cli/commands/DatasetsCommand.java

@@ -23,6 +23,7 @@ import com.uber.hoodie.cli.HoodiePrintHelper;
 import com.uber.hoodie.cli.TableHeader;
 import com.uber.hoodie.common.model.HoodieTableType;
 import com.uber.hoodie.common.table.HoodieTableMetaClient;
+import com.uber.hoodie.common.util.ConsistencyGuardConfig;
 import com.uber.hoodie.exception.DatasetNotFoundException;
 import java.io.IOException;
 import java.util.ArrayList;
@@ -39,11 +40,25 @@ public class DatasetsCommand implements CommandMarker {
 
   @CliCommand(value = "connect", help = "Connect to a hoodie dataset")
   public String connect(
-      @CliOption(key = {"path"}, mandatory = true, help = "Base Path of the dataset") final String path)
-      throws IOException {
-    boolean initialized = HoodieCLI.initConf();
-    HoodieCLI.initFS(initialized);
-    HoodieCLI.setTableMetadata(new HoodieTableMetaClient(HoodieCLI.conf, path));
+      @CliOption(key = {"path"}, mandatory = true, help = "Base Path of the dataset") final String path,
+      @CliOption(key = {"eventuallyConsistent"}, mandatory = false, unspecifiedDefaultValue = "false",
+          help = "Enable eventual consistency") final boolean eventuallyConsistent,
+      @CliOption(key = {"initialCheckIntervalMs"}, mandatory = false, unspecifiedDefaultValue = "2000",
+          help = "Initial wait time for eventual consistency") final Integer initialConsistencyIntervalMs,
+      @CliOption(key = {"maxCheckIntervalMs"}, mandatory = false, unspecifiedDefaultValue = "300000",
+          help = "Max wait time for eventual consistency") final Integer maxConsistencyIntervalMs,
+      @CliOption(key = {"maxCheckIntervalMs"}, mandatory = false, unspecifiedDefaultValue = "7",
+          help = "Max checks for eventual consistency") final Integer maxConsistencyChecks) throws IOException {
+    HoodieCLI.setConsistencyGuardConfig(
+        ConsistencyGuardConfig.newBuilder()
+            .withConsistencyCheckEnabled(eventuallyConsistent)
+            .withInitialConsistencyCheckIntervalMs(initialConsistencyIntervalMs)
+            .withMaxConsistencyCheckIntervalMs(maxConsistencyIntervalMs)
+            .withMaxConsistencyChecks(maxConsistencyChecks)
+            .build());
+    HoodieCLI.initConf();
+    HoodieCLI.connectTo(path);
+    HoodieCLI.initFS(true);
     HoodieCLI.state = HoodieCLI.CLIState.DATASET;
     return "Metadata for table " + HoodieCLI.tableMetadata.getTableConfig().getTableName() + " loaded";
   }
@@ -85,7 +100,7 @@ public class DatasetsCommand implements CommandMarker {
     HoodieTableMetaClient.initTableType(HoodieCLI.conf, path, tableType, name, payloadClass);
 
     // Now connect to ensure loading works
-    return connect(path);
+    return connect(path, false, 0, 0, 0);
   }
 
   @CliAvailabilityIndicator({"desc"})

hoodie-cli/src/main/java/com/uber/hoodie/cli/commands/SavepointsCommand.java

@@ -23,7 +23,6 @@ import com.uber.hoodie.cli.HoodieCLI;
 import com.uber.hoodie.cli.HoodiePrintHelper;
 import com.uber.hoodie.cli.utils.InputStreamConsumer;
 import com.uber.hoodie.cli.utils.SparkUtil;
-import com.uber.hoodie.common.table.HoodieTableMetaClient;
 import com.uber.hoodie.common.table.HoodieTimeline;
 import com.uber.hoodie.common.table.timeline.HoodieActiveTimeline;
 import com.uber.hoodie.common.table.timeline.HoodieInstant;
@@ -133,9 +132,8 @@ public class SavepointsCommand implements CommandMarker {
 
   @CliCommand(value = "savepoints refresh", help = "Refresh the savepoints")
   public String refreshMetaClient() throws IOException {
-    HoodieTableMetaClient metadata = new HoodieTableMetaClient(HoodieCLI.conf, HoodieCLI.tableMetadata.getBasePath());
-    HoodieCLI.setTableMetadata(metadata);
-    return "Metadata for table " + metadata.getTableConfig().getTableName() + " refreshed.";
+    HoodieCLI.refreshTableMetadata();
+    return "Metadata for table " + HoodieCLI.tableMetadata.getTableConfig().getTableName() + " refreshed.";
   }
 
   private static HoodieWriteClient createHoodieClient(JavaSparkContext jsc, String basePath) throws Exception {