[HUDI-1014] Adding Upgrade and downgrade infra for smooth transitioning from list based rollback to marker based rollback (#1858)

- This pull request adds upgrade/downgrade infra for smooth transition from list based rollback to marker based rollback*
 - A new property called hoodie.table.version is added to hoodie.properties file as part of this. Whenever hoodie is launched with newer table version i.e 1(or moving from pre 0.6.0 to 0.6.0), an upgrade step will be executed automatically to adhere to marker based rollback.*
 - This automatic upgrade step will happen just once per dataset as the hoodie.table.version will be updated in property file after upgrade is completed once*
 - Similarly, a command line tool for Downgrading is added if incase some user wants to downgrade hoodie from table version 1 to 0 or move from hoodie 0.6.0 to pre 0.6.0*
 - *Added UpgradeDowngrade to assist in upgrading or downgrading hoodie table*
 - *Added Interfaces for upgrade and downgrade and concrete implementations for upgrading from 0 to 1 and downgrading from 1 to 0.*
 - *Made some changes to ListingBasedRollbackHelper to expose just rollback stats w/o performing actual rollback, which will be consumed by Upgrade infra*
- Reworking failure handling for upgrade/downgrade
 - Changed tests accordingly, added one test around left over cleanup
 - New tables now write table version into hoodie.properties
 - Clean up code naming, abstractions.

Co-authored-by: Vinoth Chandar <vinoth@apache.org>

hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableConfig.java

@@ -24,6 +24,7 @@ import org.apache.hudi.common.model.HoodieTableType;
 import org.apache.hudi.common.model.OverwriteWithLatestAvroPayload;
 import org.apache.hudi.common.table.timeline.versioning.TimelineLayoutVersion;
 import org.apache.hudi.common.util.Option;
+import org.apache.hudi.common.util.ValidationUtils;
 import org.apache.hudi.exception.HoodieIOException;
 
 import org.apache.hadoop.fs.FSDataInputStream;
@@ -41,9 +42,8 @@ import java.util.Properties;
 import java.util.stream.Collectors;
 
 /**
- * Configurations on the Hoodie Table like type of ingestion, storage formats, hive table name etc Configurations are
- * loaded from hoodie.properties, these properties are usually set during initializing a path as hoodie base path and
- * never changes during the lifetime of a hoodie table.
+ * Configurations on the Hoodie Table like type of ingestion, storage formats, hive table name etc Configurations are loaded from hoodie.properties, these properties are usually set during
+ * initializing a path as hoodie base path and never changes during the lifetime of a hoodie table.
  *
  * @see HoodieTableMetaClient
  * @since 0.3.0
@@ -55,6 +55,7 @@ public class HoodieTableConfig implements Serializable {
   public static final String HOODIE_PROPERTIES_FILE = "hoodie.properties";
   public static final String HOODIE_TABLE_NAME_PROP_NAME = "hoodie.table.name";
   public static final String HOODIE_TABLE_TYPE_PROP_NAME = "hoodie.table.type";
+  public static final String HOODIE_TABLE_VERSION_PROP_NAME = "hoodie.table.version";
   @Deprecated
   public static final String HOODIE_RO_FILE_FORMAT_PROP_NAME = "hoodie.table.ro.file.format";
   @Deprecated
@@ -68,13 +69,13 @@ public class HoodieTableConfig implements Serializable {
   public static final String HOODIE_BOOTSTRAP_BASE_PATH = "hoodie.bootstrap.base.path";
 
   public static final HoodieTableType DEFAULT_TABLE_TYPE = HoodieTableType.COPY_ON_WRITE;
+  public static final HoodieTableVersion DEFAULT_TABLE_VERSION = HoodieTableVersion.ZERO;
   public static final HoodieFileFormat DEFAULT_BASE_FILE_FORMAT = HoodieFileFormat.PARQUET;
   public static final HoodieFileFormat DEFAULT_LOG_FILE_FORMAT = HoodieFileFormat.HOODIE_LOG;
   public static final String DEFAULT_PAYLOAD_CLASS = OverwriteWithLatestAvroPayload.class.getName();
   public static final String DEFAULT_BOOTSTRAP_INDEX_CLASS = HFileBootstrapIndex.class.getName();
-
-  public static final Integer DEFAULT_TIMELINE_LAYOUT_VERSION = TimelineLayoutVersion.VERSION_0;
   public static final String DEFAULT_ARCHIVELOG_FOLDER = "";
+
   private Properties props;
 
   public HoodieTableConfig(FileSystem fs, String metaPath, String payloadClassName) {
@@ -96,6 +97,8 @@ public class HoodieTableConfig implements Serializable {
       throw new HoodieIOException("Could not load Hoodie properties from " + propertyPath, e);
     }
     this.props = props;
+    ValidationUtils.checkArgument(props.containsKey(HOODIE_TABLE_TYPE_PROP_NAME) && props.containsKey(HOODIE_TABLE_NAME_PROP_NAME),
+        "hoodie.properties file seems invalid. Please check for left over `.updated` files if any, manually copy it to hoodie.properties and retry");
   }
 
   public HoodieTableConfig(Properties props) {
@@ -107,7 +110,8 @@ public class HoodieTableConfig implements Serializable {
    *
    * @deprecated
    */
-  public HoodieTableConfig() {}
+  public HoodieTableConfig() {
+  }
 
   /**
    * Initialize the hoodie meta directory and any necessary files inside the meta (including the hoodie.properties).
@@ -160,6 +164,19 @@ public class HoodieTableConfig implements Serializable {
         : Option.empty();
   }
 
+  /**
+   * @return the hoodie.table.version from hoodie.properties file.
+   */
+  public HoodieTableVersion getTableVersion() {
+    return props.containsKey(HOODIE_TABLE_VERSION_PROP_NAME)
+        ? HoodieTableVersion.versionFromCode(Integer.parseInt(props.getProperty(HOODIE_TABLE_VERSION_PROP_NAME)))
+        : DEFAULT_TABLE_VERSION;
+  }
+
+  public void setTableVersion(HoodieTableVersion tableVersion) {
+    props.put(HOODIE_TABLE_VERSION_PROP_NAME, Integer.toString(tableVersion.versionCode()));
+  }
+
   /**
    * Read the payload class for HoodieRecords from the table properties.
    */
@@ -231,4 +248,8 @@ public class HoodieTableConfig implements Serializable {
     return props.entrySet().stream()
         .collect(Collectors.toMap(e -> String.valueOf(e.getKey()), e -> String.valueOf(e.getValue())));
   }
+
+  public Properties getProperties() {
+    return props;
+  }
 }

hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableMetaClient.java

@@ -114,7 +114,8 @@ public class HoodieTableMetaClient implements Serializable {
   }
 
   public HoodieTableMetaClient(Configuration conf, String basePath, boolean loadActiveTimelineOnLoad,
-      ConsistencyGuardConfig consistencyGuardConfig, Option<TimelineLayoutVersion> layoutVersion, String payloadClassName) {
+                               ConsistencyGuardConfig consistencyGuardConfig, Option<TimelineLayoutVersion> layoutVersion,
+                               String payloadClassName) {
     LOG.info("Loading HoodieTableMetaClient from " + basePath);
     this.basePath = basePath;
     this.consistencyGuardConfig = consistencyGuardConfig;
@@ -362,6 +363,7 @@ public class HoodieTableMetaClient implements Serializable {
     Properties properties = new Properties();
     properties.setProperty(HoodieTableConfig.HOODIE_TABLE_NAME_PROP_NAME, tableName);
     properties.setProperty(HoodieTableConfig.HOODIE_TABLE_TYPE_PROP_NAME, tableType.name());
+    properties.setProperty(HoodieTableConfig.HOODIE_TABLE_VERSION_PROP_NAME, String.valueOf(HoodieTableVersion.current().versionCode()));
     if (tableType == HoodieTableType.MERGE_ON_READ && payloadClassName != null) {
       properties.setProperty(HoodieTableConfig.HOODIE_PAYLOAD_CLASS_PROP_NAME, payloadClassName);
     }
@@ -614,8 +616,4 @@ public class HoodieTableMetaClient implements Serializable {
   public void setActiveTimeline(HoodieActiveTimeline activeTimeline) {
     this.activeTimeline = activeTimeline;
   }
-
-  public void setTableConfig(HoodieTableConfig tableConfig) {
-    this.tableConfig = tableConfig;
-  }
 }

hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableVersion.java

@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.common.table;
+
+import org.apache.hudi.exception.HoodieException;
+
+import java.util.Arrays;
+
+/**
+ * Table's version that controls what version of writer/readers can actually read/write
+ * to a given table.
+ */
+public enum HoodieTableVersion {
+  // < 0.6.0 versions
+  ZERO(0),
+  // 0.6.0 onwards
+  ONE(1);
+
+  private final int versionCode;
+
+  HoodieTableVersion(int versionCode) {
+    this.versionCode = versionCode;
+  }
+
+  public int versionCode() {
+    return versionCode;
+  }
+
+  public static HoodieTableVersion current() {
+    return ONE;
+  }
+
+  static HoodieTableVersion versionFromCode(int versionCode) {
+    return Arrays.stream(HoodieTableVersion.values())
+        .filter(v -> v.versionCode == versionCode).findAny()
+        .orElseThrow(() -> new HoodieException("Unknown versionCode:" + versionCode));
+  }
+}