[HUDI-1138] Add timeline-server-based marker file strategy for improving marker-related latency (#3233)

- Can be enabled for cloud stores like S3. Not supported for hdfs yet, due to partial write failures.

hudi-common/src/main/java/org/apache/hudi/common/table/marker/MarkerOperation.java

@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.common.table.marker;
+
+import java.io.Serializable;
+
+/**
+ * Stores URLs to timeline server for marker-related operations.
+ */
+public class MarkerOperation implements Serializable {
+
+  private static final String BASE_URL = "/v1/hoodie/marker";
+
+  public static final String MARKER_DIR_PATH_PARAM = "markerdirpath";
+  public static final String MARKER_NAME_PARAM = "markername";
+
+  // GET requests
+  public static final String ALL_MARKERS_URL = String.format("%s/%s", BASE_URL, "all");
+  public static final String CREATE_AND_MERGE_MARKERS_URL = String.format("%s/%s", BASE_URL, "create-and-merge");
+  public static final String MARKERS_DIR_EXISTS_URL = String.format("%s/%s", BASE_URL, "dir/exists");
+
+  // POST requests
+  public static final String CREATE_MARKER_URL = String.format("%s/%s", BASE_URL, "create");
+  public static final String DELETE_MARKER_DIR_URL = String.format("%s/%s", BASE_URL, "dir/delete");
+}

hudi-common/src/main/java/org/apache/hudi/common/util/FileIOUtils.java

@@ -18,7 +18,11 @@
 
 package org.apache.hudi.common.util;
 
+import org.apache.log4j.LogManager;
+import org.apache.log4j.Logger;
+
 import java.io.ByteArrayOutputStream;
+import java.io.Closeable;
 import java.io.File;
 import java.io.FileOutputStream;
 import java.io.IOException;
@@ -34,7 +38,7 @@ import java.util.Comparator;
  * Bunch of utility methods for working with files and byte streams.
  */
 public class FileIOUtils {
-
+  public static final Logger LOG = LogManager.getLogger(FileIOUtils.class);
   public static final long KB = 1024;
 
   public static void deleteDirectory(File directory) throws IOException {
@@ -91,4 +95,20 @@ public class FileIOUtils {
     out.flush();
     out.close();
   }
+
+  /**
+   * Closes {@code Closeable} quietly.
+   *
+   * @param closeable {@code Closeable} to close
+   */
+  public static void closeQuietly(Closeable closeable) {
+    if (closeable == null) {
+      return;
+    }
+    try {
+      closeable.close();
+    } catch (IOException e) {
+      LOG.warn("IOException during close", e);
+    }
+  }
 }

hudi-common/src/main/java/org/apache/hudi/exception/HoodieRemoteException.java

@@ -29,4 +29,7 @@ public class HoodieRemoteException extends RuntimeException {
     super(t.getMessage(), t);
   }
 
+  public HoodieRemoteException(String message, IOException t) {
+    super(message + "\n" + t.getMessage(), t);
+  }
 }