Refactor HoodieTable Rollback to write one rollback instant for a batch of commits to rollback

hoodie-common/src/main/avro/HoodieRestoreMetadata.avsc

@@ -0,0 +1,17 @@
+{"namespace": "com.uber.hoodie.avro.model",
+ "type": "record",
+ "name": "HoodieRestoreMetadata",
+ "fields": [
+     {"name": "startRestoreTime", "type": "string"},
+     {"name": "timeTakenInMillis", "type": "long"},
+     {"name": "instantsToRollback", "type": {"type": "array", "items": "string"}},
+     {"name": "hoodieRestoreMetadata", "type": {
+     "type" : "map", "values" : {
+        "type": "array",
+        "default": "null",
+        "items": "HoodieRollbackMetadata",
+        "name": "hoodieRollbackMetadata"
+     }
+   }}
+ ]
+}

hoodie-common/src/main/java/com/uber/hoodie/common/table/HoodieTimeline.java

@@ -48,6 +48,7 @@ public interface HoodieTimeline extends Serializable {
   // (compaction-requested), (compaction-inflight), (completed)
   String COMPACTION_ACTION = "compaction";
   String REQUESTED_EXTENSION = ".requested";
+  String RESTORE_ACTION = "restore";
 
   String COMMIT_EXTENSION = "." + COMMIT_ACTION;
   String DELTA_COMMIT_EXTENSION = "." + DELTA_COMMIT_ACTION;
@@ -66,6 +67,8 @@ public interface HoodieTimeline extends Serializable {
       StringUtils.join(".", REQUESTED_COMPACTION_SUFFIX);
   String INFLIGHT_COMPACTION_EXTENSION =
       StringUtils.join(".", COMPACTION_ACTION, INFLIGHT_EXTENSION);
+  String INFLIGHT_RESTORE_EXTENSION = "." + RESTORE_ACTION + INFLIGHT_EXTENSION;
+  String RESTORE_EXTENSION = "." + RESTORE_ACTION;
 
   /**
    * Filter this timeline to just include the in-flights
@@ -248,6 +251,14 @@ public interface HoodieTimeline extends Serializable {
     return StringUtils.join(commitTime, HoodieTimeline.REQUESTED_COMPACTION_EXTENSION);
   }
 
+  static String makeRestoreFileName(String instant) {
+    return StringUtils.join(instant, HoodieTimeline.RESTORE_EXTENSION);
+  }
+
+  static String makeInflightRestoreFileName(String instant) {
+    return StringUtils.join(instant, HoodieTimeline.INFLIGHT_RESTORE_EXTENSION);
+  }
+
   static String makeDeltaFileName(String commitTime) {
     return commitTime + HoodieTimeline.DELTA_COMMIT_EXTENSION;
   }

hoodie-common/src/main/java/com/uber/hoodie/common/table/timeline/HoodieActiveTimeline.java

@@ -54,7 +54,8 @@ public class HoodieActiveTimeline extends HoodieDefaultTimeline {
   public static final Set<String> VALID_EXTENSIONS_IN_ACTIVE_TIMELINE = new HashSet<>(Arrays.asList(
       new String[]{COMMIT_EXTENSION, INFLIGHT_COMMIT_EXTENSION, DELTA_COMMIT_EXTENSION,
           INFLIGHT_DELTA_COMMIT_EXTENSION, SAVEPOINT_EXTENSION, INFLIGHT_SAVEPOINT_EXTENSION,
-          CLEAN_EXTENSION, INFLIGHT_CLEAN_EXTENSION, INFLIGHT_COMPACTION_EXTENSION, REQUESTED_COMPACTION_EXTENSION}));
+          CLEAN_EXTENSION, INFLIGHT_CLEAN_EXTENSION, INFLIGHT_COMPACTION_EXTENSION, REQUESTED_COMPACTION_EXTENSION,
+          INFLIGHT_RESTORE_EXTENSION, RESTORE_EXTENSION}));
 
   private static final transient Logger log = LogManager.getLogger(HoodieActiveTimeline.class);
   private HoodieTableMetaClient metaClient;
@@ -186,6 +187,14 @@ public class HoodieActiveTimeline extends HoodieDefaultTimeline {
         (Function<HoodieInstant, Optional<byte[]>> & Serializable) this::getInstantDetails);
   }
 
+  /**
+   * Get only the restore action (inflight and completed) in the active timeline
+   */
+  public HoodieTimeline getRestoreTimeline() {
+    return new HoodieDefaultTimeline(filterInstantsByAction(RESTORE_ACTION),
+        (Function<HoodieInstant, Optional<byte[]>> & Serializable) this::getInstantDetails);
+  }
+
 
   protected Stream<HoodieInstant> filterInstantsByAction(String action) {
     return instants.stream().filter(s -> s.getAction().equals(action));

hoodie-common/src/main/java/com/uber/hoodie/common/table/timeline/HoodieInstant.java

@@ -131,6 +131,9 @@ public class HoodieInstant implements Serializable {
       } else {
         return HoodieTimeline.makeCommitFileName(timestamp);
       }
+    } else if (HoodieTimeline.RESTORE_ACTION.equals(action)) {
+      return isInflight() ? HoodieTimeline.makeInflightRestoreFileName(timestamp)
+          : HoodieTimeline.makeRestoreFileName(timestamp);
     }
     throw new IllegalArgumentException("Cannot get file name for unknown action " + action);
   }

hoodie-common/src/main/java/com/uber/hoodie/common/util/AvroUtils.java

@@ -22,6 +22,7 @@ import com.google.common.collect.Lists;
 import com.uber.hoodie.avro.model.HoodieCleanMetadata;
 import com.uber.hoodie.avro.model.HoodieCleanPartitionMetadata;
 import com.uber.hoodie.avro.model.HoodieCompactionPlan;
+import com.uber.hoodie.avro.model.HoodieRestoreMetadata;
 import com.uber.hoodie.avro.model.HoodieRollbackMetadata;
 import com.uber.hoodie.avro.model.HoodieRollbackPartitionMetadata;
 import com.uber.hoodie.avro.model.HoodieSavepointMetadata;
@@ -34,6 +35,7 @@ import com.uber.hoodie.common.model.HoodieRecord;
 import com.uber.hoodie.exception.HoodieIOException;
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
+import java.util.Arrays;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
@@ -114,16 +116,28 @@ public class AvroUtils {
         totalDeleted, earliestCommitToRetain, partitionMetadataBuilder.build());
   }
 
+  public static HoodieRestoreMetadata convertRestoreMetadata(String startRestoreTime,
+      Optional<Long> durationInMs, List<String> commits, Map<String, List<HoodieRollbackStat>> commitToStats) {
+    ImmutableMap.Builder<String, List<HoodieRollbackMetadata>> commitToStatBuilder = ImmutableMap.builder();
+    for (Map.Entry<String, List<HoodieRollbackStat>> commitToStat : commitToStats.entrySet()) {
+      commitToStatBuilder.put(commitToStat.getKey(), Arrays.asList(convertRollbackMetadata(startRestoreTime,
+          durationInMs, commits, commitToStat.getValue())));
+    }
+    return new HoodieRestoreMetadata(startRestoreTime, durationInMs.orElseGet(() -> -1L), commits,
+        commitToStatBuilder.build());
+  }
+
   public static HoodieRollbackMetadata convertRollbackMetadata(String startRollbackTime,
-      Optional<Long> durationInMs, List<String> commits, List<HoodieRollbackStat> stats) {
+      Optional<Long> durationInMs, List<String> commits, List<HoodieRollbackStat> rollbackStats) {
     ImmutableMap.Builder<String, HoodieRollbackPartitionMetadata> partitionMetadataBuilder =
         ImmutableMap.builder();
     int totalDeleted = 0;
-    for (HoodieRollbackStat stat : stats) {
+    for (HoodieRollbackStat stat : rollbackStats) {
       HoodieRollbackPartitionMetadata metadata =
           new HoodieRollbackPartitionMetadata(stat.getPartitionPath(),
               stat.getSuccessDeleteFiles(), stat.getFailedDeleteFiles());
-      partitionMetadataBuilder.put(stat.getPartitionPath(), metadata);
+      partitionMetadataBuilder
+          .put(stat.getPartitionPath(), metadata);
       totalDeleted += stat.getSuccessDeleteFiles().size();
     }
     return new HoodieRollbackMetadata(startRollbackTime, durationInMs.orElseGet(() -> -1L),
@@ -163,6 +177,11 @@ public class AvroUtils {
     return serializeAvroMetadata(rollbackMetadata, HoodieRollbackMetadata.class);
   }
 
+  public static Optional<byte[]> serializeRestoreMetadata(
+      HoodieRestoreMetadata restoreMetadata) throws IOException {
+    return serializeAvroMetadata(restoreMetadata, HoodieRestoreMetadata.class);
+  }
+
   public static <T extends SpecificRecordBase> Optional<byte[]> serializeAvroMetadata(T metadata,
       Class<T> clazz) throws IOException {
     DatumWriter<T> datumWriter = new SpecificDatumWriter<>(clazz);

hoodie-common/src/main/java/com/uber/hoodie/common/util/FSUtils.java

@@ -398,6 +398,20 @@ public class FSUtils {
     });
   }
 
+  public static void deleteOlderRestoreMetaFiles(FileSystem fs, String metaPath,
+      Stream<HoodieInstant> instants) {
+    //TODO - this should be archived when archival is made general for all meta-data
+    // skip MIN_ROLLBACK_TO_KEEP and delete rest
+    instants.skip(MIN_ROLLBACK_TO_KEEP).map(s -> {
+      try {
+        return fs.delete(new Path(metaPath, s.getFileName()), false);
+      } catch (IOException e) {
+        throw new HoodieIOException(
+            "Could not delete restore meta files " + s.getFileName(), e);
+      }
+    });
+  }
+
   public static void createPathIfNotExists(FileSystem fs, Path partitionPath) throws IOException {
     if (!fs.exists(partitionPath)) {
       fs.mkdirs(partitionPath);