[HUDI-1292] Created a config to enable/disable syncing of metadata table. (#3427)

* [HUDI-1292] Created a config to enable/disable syncing of metadata table.

- Metadata Table should only be synced from a single pipeline to prevent conflicts.
- Skip syncing metadata table for clustering and compaction
- Renamed useFileListingMetadata

Co-authored-by: Vinoth Chandar <vinoth@apache.org>

hudi-common/src/main/java/org/apache/hudi/common/config/HoodieMetadataConfig.java

@@ -44,6 +44,13 @@ public final class HoodieMetadataConfig extends HoodieConfig {
       .sinceVersion("0.7.0")
       .withDocumentation("Enable the internal metadata table which serves table metadata like level file listings");
 
+  // Enable syncing the Metadata Table
+  public static final ConfigProperty<Boolean> METADATA_SYNC_ENABLE_PROP = ConfigProperty
+      .key(METADATA_PREFIX + ".sync.enable")
+      .defaultValue(true)
+      .sinceVersion("0.9.0")
+      .withDocumentation("Enable syncing of metadata table from actions on the dataset");
+
   // Validate contents of Metadata Table on each access against the actual filesystem
   public static final ConfigProperty<Boolean> METADATA_VALIDATE_PROP = ConfigProperty
       .key(METADATA_PREFIX + ".validate")
@@ -137,10 +144,14 @@ public final class HoodieMetadataConfig extends HoodieConfig {
     return getBoolean(HoodieMetadataConfig.HOODIE_ASSUME_DATE_PARTITIONING_PROP);
   }
 
-  public boolean useFileListingMetadata() {
+  public boolean enabled() {
     return getBoolean(METADATA_ENABLE_PROP);
   }
 
+  public boolean enableSync() {
+    return enabled() && getBoolean(HoodieMetadataConfig.METADATA_SYNC_ENABLE_PROP);
+  }
+
   public boolean validateFileListingMetadata() {
     return getBoolean(METADATA_VALIDATE_PROP);
   }
@@ -174,6 +185,11 @@ public final class HoodieMetadataConfig extends HoodieConfig {
       return this;
     }
 
+    public Builder enableSync(boolean enable) {
+      metadataConfig.setValue(METADATA_SYNC_ENABLE_PROP, String.valueOf(enable));
+      return this;
+    }
+
     public Builder enableMetrics(boolean enableMetrics) {
       metadataConfig.setValue(METADATA_METRICS_ENABLE_PROP, String.valueOf(enableMetrics));
       return this;

hudi-common/src/main/java/org/apache/hudi/common/table/view/FileSystemViewManager.java

@@ -160,7 +160,7 @@ public class FileSystemViewManager {
                                                                         HoodieTableMetaClient metaClient, SerializableSupplier<HoodieTableMetadata> metadataSupplier) {
     LOG.info("Creating InMemory based view for basePath " + metaClient.getBasePath());
     HoodieTimeline timeline = metaClient.getActiveTimeline().filterCompletedAndCompactionInstants();
-    if (metadataConfig.useFileListingMetadata()) {
+    if (metadataConfig.enabled()) {
       ValidationUtils.checkArgument(metadataSupplier != null, "Metadata supplier is null. Cannot instantiate metadata file system view");
       return new HoodieMetadataFileSystemView(metaClient, metaClient.getActiveTimeline().getCommitsTimeline().filterCompletedInstants(),
           metadataSupplier.get());
@@ -181,7 +181,7 @@ public class FileSystemViewManager {
                                                                                    HoodieMetadataConfig metadataConfig,
                                                                                    HoodieTimeline timeline) {
     LOG.info("Creating InMemory based view for basePath " + metaClient.getBasePath());
-    if (metadataConfig.useFileListingMetadata()) {
+    if (metadataConfig.enabled()) {
       return new HoodieMetadataFileSystemView(engineContext, metaClient, timeline, metadataConfig);
     }
     return new HoodieTableFileSystemView(metaClient, timeline);

hudi-common/src/main/java/org/apache/hudi/metadata/BaseTableMetadata.java

@@ -78,7 +78,7 @@ public abstract class BaseTableMetadata implements HoodieTableMetadata {
     this.spillableMapDirectory = spillableMapDirectory;
     this.metadataConfig = metadataConfig;
 
-    this.enabled = metadataConfig.useFileListingMetadata();
+    this.enabled = metadataConfig.enabled();
     if (metadataConfig.enableMetrics()) {
       this.metrics = Option.of(new HoodieMetadataMetrics(Registry.getRegistry("HoodieMetadata")));
     } else {

hudi-common/src/main/java/org/apache/hudi/metadata/HoodieTableMetadata.java

@@ -78,7 +78,7 @@ public interface HoodieTableMetadata extends Serializable, AutoCloseable {
 
   static HoodieTableMetadata create(HoodieEngineContext engineContext, HoodieMetadataConfig metadataConfig, String datasetBasePath,
                                     String spillableMapPath, boolean reuse) {
-    if (metadataConfig.useFileListingMetadata()) {
+    if (metadataConfig.enabled()) {
       return new HoodieBackedTableMetadata(engineContext, metadataConfig, datasetBasePath, spillableMapPath, reuse);
     } else {
       return new FileSystemBackedTableMetadata(engineContext, new SerializableConfiguration(engineContext.getHadoopConf()),