[HUDI-2931] Add config to disable table services (#4777)

hudi-client/hudi-client-common/src/main/java/org/apache/hudi/async/AsyncArchiveService.java

@@ -34,7 +34,7 @@ import java.util.concurrent.Executors;
 /**
  * Async archive service to run concurrently with write operation.
  */
-public class AsyncArchiveService extends HoodieAsyncService {
+public class AsyncArchiveService extends HoodieAsyncTableService {
 
   private static final Logger LOG = LogManager.getLogger(AsyncArchiveService.class);
 
@@ -42,6 +42,7 @@ public class AsyncArchiveService extends HoodieAsyncService {
   private final transient ExecutorService executor = Executors.newSingleThreadExecutor();
 
   protected AsyncArchiveService(BaseHoodieWriteClient writeClient) {
+    super(writeClient.getConfig());
     this.writeClient = writeClient;
   }
 

hudi-client/hudi-client-common/src/main/java/org/apache/hudi/async/AsyncCleanerService.java

@@ -35,7 +35,7 @@ import java.util.concurrent.Executors;
 /**
  * Async clean service to run concurrently with write operation.
  */
-public class AsyncCleanerService extends HoodieAsyncService {
+public class AsyncCleanerService extends HoodieAsyncTableService {
 
   private static final Logger LOG = LogManager.getLogger(AsyncCleanerService.class);
 
@@ -43,6 +43,7 @@ public class AsyncCleanerService extends HoodieAsyncService {
   private final transient ExecutorService executor = Executors.newSingleThreadExecutor();
 
   protected AsyncCleanerService(BaseHoodieWriteClient writeClient) {
+    super(writeClient.getConfig());
     this.writeClient = writeClient;
   }
 

hudi-client/hudi-client-common/src/main/java/org/apache/hudi/async/AsyncClusteringService.java

@@ -38,7 +38,7 @@ import java.util.stream.IntStream;
  * Async clustering service that runs in a separate thread.
  * Currently, only one clustering thread is allowed to run at any time.
  */
-public abstract class AsyncClusteringService extends HoodieAsyncService {
+public abstract class AsyncClusteringService extends HoodieAsyncTableService {
 
   private static final long serialVersionUID = 1L;
   private static final Logger LOG = LogManager.getLogger(AsyncClusteringService.class);
@@ -51,7 +51,7 @@ public abstract class AsyncClusteringService extends HoodieAsyncService {
   }
 
   public AsyncClusteringService(BaseHoodieWriteClient writeClient, boolean runInDaemonMode) {
-    super(runInDaemonMode);
+    super(writeClient.getConfig(), runInDaemonMode);
     this.clusteringClient = createClusteringClient(writeClient);
     this.maxConcurrentClustering = 1;
   }

hudi-client/hudi-client-common/src/main/java/org/apache/hudi/async/AsyncCompactService.java

@@ -37,7 +37,7 @@ import java.util.stream.IntStream;
 /**
  * Async Compactor Service that runs in separate thread. Currently, only one compactor is allowed to run at any time.
  */
-public abstract class AsyncCompactService extends HoodieAsyncService {
+public abstract class AsyncCompactService extends HoodieAsyncTableService {
 
   private static final long serialVersionUID = 1L;
   private static final Logger LOG = LogManager.getLogger(AsyncCompactService.class);
@@ -56,7 +56,7 @@ public abstract class AsyncCompactService extends HoodieAsyncService {
   }
 
   public AsyncCompactService(HoodieEngineContext context, BaseHoodieWriteClient client, boolean runInDaemonMode) {
-    super(runInDaemonMode);
+    super(client.getConfig(), runInDaemonMode);
     this.context = context;
     this.compactor = createCompactor(client);
     this.maxConcurrentCompaction = 1;

hudi-client/hudi-client-common/src/main/java/org/apache/hudi/async/HoodieAsyncService.java

@@ -89,6 +89,9 @@ public abstract class HoodieAsyncService implements Serializable {
    * @throws InterruptedException
    */
   public void waitForShutdown() throws ExecutionException, InterruptedException {
+    if (future == null) {
+      return;
+    }
     try {
       future.get();
     } catch (ExecutionException ex) {
@@ -152,6 +155,9 @@ public abstract class HoodieAsyncService implements Serializable {
    */
   @SuppressWarnings("unchecked")
   private void shutdownCallback(Function<Boolean, Boolean> callback) {
+    if (future == null) {
+      return;
+    }
     future.whenComplete((resp, error) -> {
       if (null != callback) {
         callback.apply(null != error);

hudi-client/hudi-client-common/src/main/java/org/apache/hudi/async/HoodieAsyncTableService.java

@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hudi.async;
+
+import org.apache.hudi.client.RunsTableService;
+import org.apache.hudi.config.HoodieWriteConfig;
+
+import java.util.function.Function;
+
+public abstract class HoodieAsyncTableService extends HoodieAsyncService implements RunsTableService {
+
+  protected HoodieWriteConfig writeConfig;
+
+  protected HoodieAsyncTableService() {
+  }
+
+  protected HoodieAsyncTableService(HoodieWriteConfig writeConfig) {
+    this.writeConfig = writeConfig;
+  }
+
+  protected HoodieAsyncTableService(HoodieWriteConfig writeConfig, boolean runInDaemonMode) {
+    super(runInDaemonMode);
+    this.writeConfig = writeConfig;
+  }
+
+  @Override
+  public void start(Function<Boolean, Boolean> onShutdownCallback) {
+    if (!tableServicesEnabled(writeConfig)) {
+      return;
+    }
+    super.start(onShutdownCallback);
+  }
+}

hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/BaseHoodieWriteClient.java

@@ -100,7 +100,8 @@ import java.util.stream.Stream;
  * @param <K> Type of keys
  * @param <O> Type of outputs
  */
-public abstract class BaseHoodieWriteClient<T extends HoodieRecordPayload, I, K, O> extends BaseHoodieClient {
+public abstract class BaseHoodieWriteClient<T extends HoodieRecordPayload, I, K, O> extends BaseHoodieClient
+    implements RunsTableService {
 
   protected static final String LOOKUP_STR = "lookup";
   private static final long serialVersionUID = 1L;
@@ -470,6 +471,9 @@ public abstract class BaseHoodieWriteClient<T extends HoodieRecordPayload, I, K,
   }
 
   protected void runTableServicesInline(HoodieTable<T, I, K, O> table, HoodieCommitMetadata metadata, Option<Map<String, String>> extraMetadata) {
+    if (!tableServicesEnabled(config)) {
+      return;
+    }
     if (config.areAnyTableServicesExecutedInline() || config.areAnyTableServicesScheduledInline()) {
       if (config.isMetadataTableEnabled()) {
         table.getHoodieView().sync();
@@ -760,6 +764,9 @@ public abstract class BaseHoodieWriteClient<T extends HoodieRecordPayload, I, K,
    * @param skipLocking if this is triggered by another parent transaction, locking can be skipped.
    */
   public HoodieCleanMetadata clean(String cleanInstantTime, boolean scheduleInline, boolean skipLocking) throws HoodieIOException {
+    if (!tableServicesEnabled(config)) {
+      return null;
+    }
     if (scheduleInline) {
       scheduleTableServiceInternal(cleanInstantTime, Option.empty(), TableServiceType.CLEAN);
     }
@@ -799,6 +806,9 @@ public abstract class BaseHoodieWriteClient<T extends HoodieRecordPayload, I, K,
    * @param table table to commit on.
    */
   protected void archive(HoodieTable<T, I, K, O> table) {
+    if (!tableServicesEnabled(config)) {
+      return;
+    }
     try {
       // We cannot have unbounded commit files. Archive commits if we have to archive
       HoodieTimelineArchiver archiver = new HoodieTimelineArchiver(config, table);
@@ -1141,7 +1151,13 @@ public abstract class BaseHoodieWriteClient<T extends HoodieRecordPayload, I, K,
 
   private Option<String> scheduleTableServiceInternal(String instantTime, Option<Map<String, String>> extraMetadata,
                                                       TableServiceType tableServiceType) {
+    if (!tableServicesEnabled(config)) {
+      return Option.empty();
+    }
     switch (tableServiceType) {
+      case ARCHIVE:
+        LOG.info("Scheduling archiving is not supported. Skipping.");
+        return Option.empty();
       case CLUSTER:
         LOG.info("Scheduling clustering at instant time :" + instantTime);
         Option<HoodieClusteringPlan> clusteringPlan = createTable(config, hadoopConf, config.isMetadataTableEnabled())

hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/RunsTableService.java

@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.client;
+
+import org.apache.hudi.config.HoodieWriteConfig;
+
+import org.apache.log4j.LogManager;
+import org.apache.log4j.Logger;
+
+public interface RunsTableService {
+
+  Logger LOG = LogManager.getLogger(RunsTableService.class);
+
+  default boolean tableServicesEnabled(HoodieWriteConfig config) {
+    boolean enabled = config.areTableServicesEnabled();
+    if (!enabled) {
+      LOG.warn(String.format("Table services are disabled. Set `%s` to enable.", HoodieWriteConfig.TABLE_SERVICES_ENABLED));
+    }
+    return enabled;
+  }
+}

hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java

@@ -61,9 +61,9 @@ import org.apache.hudi.table.RandomFileIdPrefixProvider;
 import org.apache.hudi.table.action.cluster.ClusteringPlanPartitionFilterMode;
 import org.apache.hudi.table.action.compact.CompactionTriggerStrategy;
 import org.apache.hudi.table.action.compact.strategy.CompactionStrategy;
+import org.apache.hudi.table.storage.HoodieStorageLayout;
 
 import org.apache.hadoop.hbase.io.compress.Compression;
-import org.apache.hudi.table.storage.HoodieStorageLayout;
 import org.apache.orc.CompressionKind;
 import org.apache.parquet.hadoop.metadata.CompressionCodecName;
 
@@ -440,6 +440,12 @@ public class HoodieWriteConfig extends HoodieConfig {
       .sinceVersion("0.10.0")
       .withDocumentation("File Id Prefix provider class, that implements `org.apache.hudi.fileid.FileIdPrefixProvider`");
 
+  public static final ConfigProperty<Boolean> TABLE_SERVICES_ENABLED = ConfigProperty
+      .key("hoodie.table.services.enabled")
+      .defaultValue(true)
+      .sinceVersion("0.11.0")
+      .withDocumentation("Master control to disable all table services including archive, clean, compact, cluster, etc.");
+
   private ConsistencyGuardConfig consistencyGuardConfig;
 
   // Hoodie Write Client transparently rewrites File System View config when embedded mode is enabled
@@ -1920,6 +1926,10 @@ public class HoodieWriteConfig extends HoodieConfig {
     return getString(FILEID_PREFIX_PROVIDER_CLASS);
   }
 
+  public boolean areTableServicesEnabled() {
+    return getBooleanOrDefault(TABLE_SERVICES_ENABLED);
+  }
+
   /**
    * Layout configs.
    */
@@ -2285,6 +2295,11 @@ public class HoodieWriteConfig extends HoodieConfig {
       return this;
     }
 
+    public Builder withTableServicesEnabled(boolean enabled) {
+      writeConfig.setValue(TABLE_SERVICES_ENABLED, Boolean.toString(enabled));
+      return this;
+    }
+
     public Builder withProperties(Properties properties) {
       this.writeConfig.getProps().putAll(properties);
       return this;

hudi-client/hudi-client-common/src/test/java/org/apache/hudi/async/TestHoodieAsyncTableService.java

@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hudi.async;
+
+import org.apache.hudi.common.util.collection.Pair;
+import org.apache.hudi.config.HoodieWriteConfig;
+
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.mockito.Mock;
+import org.mockito.junit.jupiter.MockitoExtension;
+
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ExecutorService;
+
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.mockito.Mockito.when;
+
+@ExtendWith(MockitoExtension.class)
+class TestHoodieAsyncTableService {
+
+  @Test
+  void tableServiceShouldNotStartIfDisabled(@Mock HoodieWriteConfig config) {
+    when(config.areTableServicesEnabled()).thenReturn(false);
+    HoodieAsyncTableService service = new DummyAsyncTableService(config);
+    service.start(null);
+    assertFalse(service.isStarted());
+  }
+
+  private static class DummyAsyncTableService extends HoodieAsyncTableService {
+
+    protected DummyAsyncTableService(HoodieWriteConfig writeConfig) {
+      super(writeConfig);
+    }
+
+    @Override
+    protected Pair<CompletableFuture, ExecutorService> startService() {
+      return null;
+    }
+  }
+}

hudi-common/src/main/java/org/apache/hudi/common/model/TableServiceType.java

@@ -24,10 +24,13 @@ import org.apache.hudi.common.table.timeline.HoodieTimeline;
  * Supported runtime table services.
  */
 public enum TableServiceType {
-  COMPACT, CLUSTER, CLEAN;
+  ARCHIVE, COMPACT, CLUSTER, CLEAN;
 
   public String getAction() {
     switch (this) {
+      case ARCHIVE:
+        // for table service type completeness; there is no timeline action associated with archive
+        return "NONE";
       case COMPACT:
         return HoodieTimeline.COMPACTION_ACTION;
       case CLEAN:

hudi-flink/src/main/java/org/apache/hudi/sink/compact/HoodieFlinkCompactor.java

@@ -18,7 +18,7 @@
 
 package org.apache.hudi.sink.compact;
 
-import org.apache.hudi.async.HoodieAsyncService;
+import org.apache.hudi.async.HoodieAsyncTableService;
 import org.apache.hudi.avro.model.HoodieCompactionPlan;
 import org.apache.hudi.client.HoodieFlinkWriteClient;
 import org.apache.hudi.common.table.HoodieTableMetaClient;
@@ -116,7 +116,7 @@ public class HoodieFlinkCompactor {
   /**
    * Schedules compaction in service.
    */
-  public static class AsyncCompactionService extends HoodieAsyncService {
+  public static class AsyncCompactionService extends HoodieAsyncTableService {
     private static final long serialVersionUID = 1L;
 
     /**
@@ -173,6 +173,7 @@ public class HoodieFlinkCompactor {
       CompactionUtil.inferChangelogMode(conf, metaClient);
 
       this.writeClient = StreamerUtil.createWriteClient(conf);
+      this.writeConfig = writeClient.getConfig();
       this.table = writeClient.getHoodieTable();
     }