From 0bd38f26ca4076dc45ee1cba4f8525550b96ce40 Mon Sep 17 00:00:00 2001 From: Raymond Xu <2701446+xushiyan@users.noreply.github.com> Date: Thu, 27 Jan 2022 17:05:08 -0800 Subject: [PATCH] [HUDI-2596] Make class names consistent in hudi-client (#4680) --- .../cli/commands/TestRollbacksCommand.java | 4 +- .../hudi/async/AsyncClusteringService.java | 14 +++--- .../hudi/async/AsyncCompactService.java | 14 +++--- .../hudi/client/AsyncCleanerService.java | 6 +-- ...usteringClient.java => BaseClusterer.java} | 8 ++-- ...tractCompactor.java => BaseCompactor.java} | 8 ++-- ...oodieClient.java => BaseHoodieClient.java} | 8 ++-- ...Client.java => BaseHoodieWriteClient.java} | 20 ++++---- .../hudi/client/CompactionAdminClient.java | 2 +- .../org/apache/hudi/keygen/KeyGenUtils.java | 8 ++-- .../TimestampBasedAvroKeyGenerator.java | 8 ++-- ...ser.java => BaseHoodieDateTimeParser.java} | 4 +- ...serImpl.java => HoodieDateTimeParser.java} | 4 +- .../HoodieBackedTableMetadataWriter.java | 6 +-- .../hudi/metrics/MetricsReporterFactory.java | 10 ++-- .../custom/CustomizableMetricsReporter.java | 46 +++++++++++++++++++ .../AbstractUserDefinedMetricsReporter.java | 37 ++++++--------- ...tHelper.java => BaseBulkInsertHelper.java} | 2 +- .../commit/BaseCommitActionExecutor.java | 2 +- ...eleteHelper.java => BaseDeleteHelper.java} | 2 +- ...tMergeHelper.java => BaseMergeHelper.java} | 2 +- ...tWriteHelper.java => BaseWriteHelper.java} | 2 +- .../hudi/table/upgrade/DowngradeHandler.java | 4 +- .../table/upgrade/OneToTwoUpgradeHandler.java | 2 +- .../upgrade/OneToZeroDowngradeHandler.java | 2 +- ...per.java => SupportsUpgradeDowngrade.java} | 2 +- .../upgrade/ThreeToTwoDowngradeHandler.java | 2 +- .../upgrade/TwoToOneDowngradeHandler.java | 2 +- .../upgrade/TwoToThreeUpgradeHandler.java | 2 +- .../hudi/table/upgrade/UpgradeDowngrade.java | 4 +- .../hudi/table/upgrade/UpgradeHandler.java | 4 +- .../upgrade/ZeroToOneUpgradeHandler.java | 2 +- .../metrics/TestMetricsReporterFactory.java | 8 ++-- .../providers/HoodieWriteClientProvider.java | 4 +- .../hudi/client/HoodieFlinkWriteClient.java | 2 +- .../FlinkHoodieBackedTableMetadataWriter.java | 2 +- .../action/commit/FlinkDeleteHelper.java | 2 +- .../table/action/commit/FlinkMergeHelper.java | 2 +- .../table/action/commit/FlinkWriteHelper.java | 2 +- .../upgrade/FlinkUpgradeDowngradeHelper.java | 2 +- .../hudi/client/HoodieJavaWriteClient.java | 2 +- .../action/commit/JavaBulkInsertHelper.java | 4 +- .../table/action/commit/JavaDeleteHelper.java | 2 +- .../table/action/commit/JavaMergeHelper.java | 2 +- .../table/action/commit/JavaWriteHelper.java | 2 +- .../async/SparkAsyncClusteringService.java | 8 ++-- .../hudi/async/SparkAsyncCompactService.java | 8 ++-- .../client/HoodieSparkClusteringClient.java | 4 +- .../hudi/client/HoodieSparkCompactor.java | 4 +- .../hudi/client/SparkRDDWriteClient.java | 2 +- .../action/commit/SparkBulkInsertHelper.java | 4 +- .../action/commit/SparkDeleteHelper.java | 4 +- .../table/action/commit/SparkMergeHelper.java | 2 +- .../table/action/commit/SparkWriteHelper.java | 4 +- ...> BaseSparkDeltaCommitActionExecutor.java} | 8 ++-- ...rkBulkInsertDeltaCommitActionExecutor.java | 10 ++-- ...nsertPreppedDeltaCommitActionExecutor.java | 8 ++-- .../SparkDeleteDeltaCommitActionExecutor.java | 4 +- .../SparkInsertDeltaCommitActionExecutor.java | 4 +- ...nsertPreppedDeltaCommitActionExecutor.java | 3 +- .../SparkUpsertDeltaCommitActionExecutor.java | 4 +- ...psertPreppedDeltaCommitActionExecutor.java | 3 +- .../upgrade/SparkUpgradeDowngradeHelper.java | 2 +- .../functional/TestHoodieBackedMetadata.java | 2 +- .../TestHoodieClientOnCopyOnWriteStorage.java | 4 +- .../table/TestHoodieMergeOnReadTable.java | 4 +- .../SparkStreamingAsyncClusteringService.java | 8 ++-- .../SparkStreamingAsyncCompactService.java | 8 ++-- 68 files changed, 216 insertions(+), 175 deletions(-) rename hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/{AbstractClusteringClient.java => BaseClusterer.java} (80%) rename hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/{AbstractCompactor.java => BaseCompactor.java} (78%) rename hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/{AbstractHoodieClient.java => BaseHoodieClient.java} (93%) rename hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/{AbstractHoodieWriteClient.java => BaseHoodieWriteClient.java} (98%) rename hudi-client/hudi-client-common/src/main/java/org/apache/hudi/keygen/parser/{AbstractHoodieDateTimeParser.java => BaseHoodieDateTimeParser.java} (94%) rename hudi-client/hudi-client-common/src/main/java/org/apache/hudi/keygen/parser/{HoodieDateTimeParserImpl.java => HoodieDateTimeParser.java} (96%) create mode 100644 hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metrics/custom/CustomizableMetricsReporter.java rename hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/commit/{AbstractBulkInsertHelper.java => BaseBulkInsertHelper.java} (96%) rename hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/commit/{AbstractDeleteHelper.java => BaseDeleteHelper.java} (95%) rename hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/commit/{AbstractMergeHelper.java => BaseMergeHelper.java} (98%) rename hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/commit/{AbstractWriteHelper.java => BaseWriteHelper.java} (97%) rename hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/{BaseUpgradeDowngradeHelper.java => SupportsUpgradeDowngrade.java} (97%) rename hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/deltacommit/{AbstractSparkDeltaCommitActionExecutor.java => BaseSparkDeltaCommitActionExecutor.java} (90%) diff --git a/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestRollbacksCommand.java b/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestRollbacksCommand.java index 17bc48f66..9a10893b3 100644 --- a/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestRollbacksCommand.java +++ b/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestRollbacksCommand.java @@ -24,7 +24,7 @@ import org.apache.hudi.cli.HoodiePrintHelper; import org.apache.hudi.cli.HoodieTableHeaderFields; import org.apache.hudi.cli.TableHeader; import org.apache.hudi.cli.functional.CLIFunctionalTestHarness; -import org.apache.hudi.client.AbstractHoodieWriteClient; +import org.apache.hudi.client.BaseHoodieWriteClient; import org.apache.hudi.client.SparkRDDWriteClient; import org.apache.hudi.common.model.HoodieTableType; import org.apache.hudi.common.table.HoodieTableMetaClient; @@ -93,7 +93,7 @@ public class TestRollbacksCommand extends CLIFunctionalTestHarness { .withRollbackUsingMarkers(false) .withIndexConfig(HoodieIndexConfig.newBuilder().withIndexType(HoodieIndex.IndexType.INMEMORY).build()).build(); - try (AbstractHoodieWriteClient client = new SparkRDDWriteClient(context(), config)) { + try (BaseHoodieWriteClient client = new SparkRDDWriteClient(context(), config)) { // Rollback inflight commit3 and commit2 client.rollback("102"); client.rollback("101"); diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/async/AsyncClusteringService.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/async/AsyncClusteringService.java index b9707bb6d..ca2faebfb 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/async/AsyncClusteringService.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/async/AsyncClusteringService.java @@ -19,8 +19,8 @@ package org.apache.hudi.async; -import org.apache.hudi.client.AbstractClusteringClient; -import org.apache.hudi.client.AbstractHoodieWriteClient; +import org.apache.hudi.client.BaseClusterer; +import org.apache.hudi.client.BaseHoodieWriteClient; import org.apache.hudi.common.table.timeline.HoodieInstant; import org.apache.hudi.common.util.collection.Pair; import org.apache.hudi.exception.HoodieIOException; @@ -44,19 +44,19 @@ public abstract class AsyncClusteringService extends HoodieAsyncService { private static final Logger LOG = LogManager.getLogger(AsyncClusteringService.class); private final int maxConcurrentClustering; - private transient AbstractClusteringClient clusteringClient; + private transient BaseClusterer clusteringClient; - public AsyncClusteringService(AbstractHoodieWriteClient writeClient) { + public AsyncClusteringService(BaseHoodieWriteClient writeClient) { this(writeClient, false); } - public AsyncClusteringService(AbstractHoodieWriteClient writeClient, boolean runInDaemonMode) { + public AsyncClusteringService(BaseHoodieWriteClient writeClient, boolean runInDaemonMode) { super(runInDaemonMode); this.clusteringClient = createClusteringClient(writeClient); this.maxConcurrentClustering = 1; } - protected abstract AbstractClusteringClient createClusteringClient(AbstractHoodieWriteClient client); + protected abstract BaseClusterer createClusteringClient(BaseHoodieWriteClient client); /** * Start clustering service. @@ -94,7 +94,7 @@ public abstract class AsyncClusteringService extends HoodieAsyncService { /** * Update the write client to be used for clustering. */ - public synchronized void updateWriteClient(AbstractHoodieWriteClient writeClient) { + public synchronized void updateWriteClient(BaseHoodieWriteClient writeClient) { this.clusteringClient.updateWriteClient(writeClient); } } diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/async/AsyncCompactService.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/async/AsyncCompactService.java index 2f6329721..3ddd88768 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/async/AsyncCompactService.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/async/AsyncCompactService.java @@ -17,8 +17,8 @@ package org.apache.hudi.async; -import org.apache.hudi.client.AbstractCompactor; -import org.apache.hudi.client.AbstractHoodieWriteClient; +import org.apache.hudi.client.BaseCompactor; +import org.apache.hudi.client.BaseHoodieWriteClient; import org.apache.hudi.common.engine.EngineProperty; import org.apache.hudi.common.engine.HoodieEngineContext; import org.apache.hudi.common.table.timeline.HoodieInstant; @@ -48,21 +48,21 @@ public abstract class AsyncCompactService extends HoodieAsyncService { public static final String COMPACT_POOL_NAME = "hoodiecompact"; private final int maxConcurrentCompaction; - private transient AbstractCompactor compactor; + private transient BaseCompactor compactor; protected transient HoodieEngineContext context; - public AsyncCompactService(HoodieEngineContext context, AbstractHoodieWriteClient client) { + public AsyncCompactService(HoodieEngineContext context, BaseHoodieWriteClient client) { this(context, client, false); } - public AsyncCompactService(HoodieEngineContext context, AbstractHoodieWriteClient client, boolean runInDaemonMode) { + public AsyncCompactService(HoodieEngineContext context, BaseHoodieWriteClient client, boolean runInDaemonMode) { super(runInDaemonMode); this.context = context; this.compactor = createCompactor(client); this.maxConcurrentCompaction = 1; } - protected abstract AbstractCompactor createCompactor(AbstractHoodieWriteClient client); + protected abstract BaseCompactor createCompactor(BaseHoodieWriteClient client); /** * Start Compaction Service. @@ -110,7 +110,7 @@ public abstract class AsyncCompactService extends HoodieAsyncService { return false; } - public synchronized void updateWriteClient(AbstractHoodieWriteClient writeClient) { + public synchronized void updateWriteClient(BaseHoodieWriteClient writeClient) { this.compactor.updateWriteClient(writeClient); } } diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/AsyncCleanerService.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/AsyncCleanerService.java index a5a38f2cc..a287ff4a6 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/AsyncCleanerService.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/AsyncCleanerService.java @@ -36,10 +36,10 @@ class AsyncCleanerService extends HoodieAsyncService { private static final Logger LOG = LogManager.getLogger(AsyncCleanerService.class); - private final AbstractHoodieWriteClient writeClient; + private final BaseHoodieWriteClient writeClient; private final transient ExecutorService executor = Executors.newSingleThreadExecutor(); - protected AsyncCleanerService(AbstractHoodieWriteClient writeClient) { + protected AsyncCleanerService(BaseHoodieWriteClient writeClient) { this.writeClient = writeClient; } @@ -53,7 +53,7 @@ class AsyncCleanerService extends HoodieAsyncService { }, executor), executor); } - public static AsyncCleanerService startAsyncCleaningIfEnabled(AbstractHoodieWriteClient writeClient) { + public static AsyncCleanerService startAsyncCleaningIfEnabled(BaseHoodieWriteClient writeClient) { AsyncCleanerService asyncCleanerService = null; if (writeClient.getConfig().isAutoClean() && writeClient.getConfig().isAsyncClean()) { asyncCleanerService = new AsyncCleanerService(writeClient); diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/AbstractClusteringClient.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/BaseClusterer.java similarity index 80% rename from hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/AbstractClusteringClient.java rename to hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/BaseClusterer.java index 34234f546..648ce805b 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/AbstractClusteringClient.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/BaseClusterer.java @@ -28,13 +28,13 @@ import java.io.Serializable; /** * Client will run one round of clustering. */ -public abstract class AbstractClusteringClient implements Serializable { +public abstract class BaseClusterer implements Serializable { private static final long serialVersionUID = 1L; - protected transient AbstractHoodieWriteClient clusteringClient; + protected transient BaseHoodieWriteClient clusteringClient; - public AbstractClusteringClient(AbstractHoodieWriteClient clusteringClient) { + public BaseClusterer(BaseHoodieWriteClient clusteringClient) { this.clusteringClient = clusteringClient; } @@ -49,7 +49,7 @@ public abstract class AbstractClusteringClient writeClient) { + public void updateWriteClient(BaseHoodieWriteClient writeClient) { this.clusteringClient = writeClient; } } diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/AbstractCompactor.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/BaseCompactor.java similarity index 78% rename from hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/AbstractCompactor.java rename to hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/BaseCompactor.java index c80b34a3e..88737dbcf 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/AbstractCompactor.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/BaseCompactor.java @@ -27,19 +27,19 @@ import java.io.Serializable; /** * Run one round of compaction. */ -public abstract class AbstractCompactor implements Serializable { +public abstract class BaseCompactor implements Serializable { private static final long serialVersionUID = 1L; - protected transient AbstractHoodieWriteClient compactionClient; + protected transient BaseHoodieWriteClient compactionClient; - public AbstractCompactor(AbstractHoodieWriteClient compactionClient) { + public BaseCompactor(BaseHoodieWriteClient compactionClient) { this.compactionClient = compactionClient; } public abstract void compact(HoodieInstant instant) throws IOException; - public void updateWriteClient(AbstractHoodieWriteClient writeClient) { + public void updateWriteClient(BaseHoodieWriteClient writeClient) { this.compactionClient = writeClient; } diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/AbstractHoodieClient.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/BaseHoodieClient.java similarity index 93% rename from hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/AbstractHoodieClient.java rename to hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/BaseHoodieClient.java index 350fe0c9b..8ea61d9a6 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/AbstractHoodieClient.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/BaseHoodieClient.java @@ -40,9 +40,9 @@ import java.io.Serializable; * Abstract class taking care of holding common member variables (FileSystem, SparkContext, HoodieConfigs) Also, manages * embedded timeline-server if enabled. */ -public abstract class AbstractHoodieClient implements Serializable, AutoCloseable { +public abstract class BaseHoodieClient implements Serializable, AutoCloseable { - private static final Logger LOG = LogManager.getLogger(AbstractHoodieClient.class); + private static final Logger LOG = LogManager.getLogger(BaseHoodieClient.class); protected final transient FileSystem fs; protected final transient HoodieEngineContext context; @@ -59,11 +59,11 @@ public abstract class AbstractHoodieClient implements Serializable, AutoCloseabl private transient Option timelineServer; private final boolean shouldStopTimelineServer; - protected AbstractHoodieClient(HoodieEngineContext context, HoodieWriteConfig clientConfig) { + protected BaseHoodieClient(HoodieEngineContext context, HoodieWriteConfig clientConfig) { this(context, clientConfig, Option.empty()); } - protected AbstractHoodieClient(HoodieEngineContext context, HoodieWriteConfig clientConfig, + protected BaseHoodieClient(HoodieEngineContext context, HoodieWriteConfig clientConfig, Option timelineServer) { this.hadoopConf = context.getHadoopConf().get(); this.fs = FSUtils.getFs(clientConfig.getBasePath(), hadoopConf); diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/AbstractHoodieWriteClient.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/BaseHoodieWriteClient.java similarity index 98% rename from hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/AbstractHoodieWriteClient.java rename to hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/BaseHoodieWriteClient.java index c9162de9e..2e04d01af 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/AbstractHoodieWriteClient.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/BaseHoodieWriteClient.java @@ -98,11 +98,11 @@ import java.util.stream.Stream; * @param Type of keys * @param Type of outputs */ -public abstract class AbstractHoodieWriteClient extends AbstractHoodieClient { +public abstract class BaseHoodieWriteClient extends BaseHoodieClient { protected static final String LOOKUP_STR = "lookup"; private static final long serialVersionUID = 1L; - private static final Logger LOG = LogManager.getLogger(AbstractHoodieWriteClient.class); + private static final Logger LOG = LogManager.getLogger(BaseHoodieWriteClient.class); protected final transient HoodieMetrics metrics; private final transient HoodieIndex index; @@ -123,7 +123,7 @@ public abstract class AbstractHoodieWriteClient timelineService) { super(context, writeConfig, timelineService); this.metrics = new HoodieMetrics(config); @@ -359,7 +359,7 @@ public abstract class AbstractHoodieWriteClient * This implementation uses sortBy (which does range partitioning based on reservoir sampling) and attempts to control - * the numbers of files with less memory compared to the {@link AbstractHoodieWriteClient#insert(I, String)} + * the numbers of files with less memory compared to the {@link BaseHoodieWriteClient#insert(I, String)} * * @param records HoodieRecords to insert * @param instantTime Instant time of the commit @@ -372,7 +372,7 @@ public abstract class AbstractHoodieWriteClient * This implementation uses sortBy (which does range partitioning based on reservoir sampling) and attempts to control - * the numbers of files with less memory compared to the {@link AbstractHoodieWriteClient#insert(I, String)}. Optionally + * the numbers of files with less memory compared to the {@link BaseHoodieWriteClient#insert(I, String)}. Optionally * it allows users to specify their own partitioner. If specified then it will be used for repartitioning records. See * {@link BulkInsertPartitioner}. * @@ -392,7 +392,7 @@ public abstract class AbstractHoodieWriteClient * This implementation uses sortBy (which does range partitioning based on reservoir sampling) and attempts to control - * the numbers of files with less memory compared to the {@link AbstractHoodieWriteClient#insert(I, String)}. Optionally + * the numbers of files with less memory compared to the {@link BaseHoodieWriteClient#insert(I, String)}. Optionally * it allows users to specify their own partitioner. If specified then it will be used for repartitioning records. See * {@link BulkInsertPartitioner}. * @@ -606,7 +606,7 @@ public abstract class AbstractHoodieWriteClient inputFormatter; private transient DateTimeFormatter partitionFormatter; - private final AbstractHoodieDateTimeParser parser; + private final BaseHoodieDateTimeParser parser; // TimeZone detailed settings reference // https://docs.oracle.com/javase/8/docs/api/java/util/TimeZone.html @@ -99,7 +99,7 @@ public class TimestampBasedAvroKeyGenerator extends SimpleAvroKeyGenerator { TimestampBasedAvroKeyGenerator(TypedProperties config, String recordKeyField, String partitionPathField) throws IOException { super(config, recordKeyField, partitionPathField); - String dateTimeParserClass = config.getString(Config.DATE_TIME_PARSER_PROP, HoodieDateTimeParserImpl.class.getName()); + String dateTimeParserClass = config.getString(Config.DATE_TIME_PARSER_PROP, HoodieDateTimeParser.class.getName()); this.parser = KeyGenUtils.createDateTimeParser(config, dateTimeParserClass); this.inputDateTimeZone = parser.getInputDateTimeZone(); this.outputDateTimeZone = parser.getOutputDateTimeZone(); diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/keygen/parser/AbstractHoodieDateTimeParser.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/keygen/parser/BaseHoodieDateTimeParser.java similarity index 94% rename from hudi-client/hudi-client-common/src/main/java/org/apache/hudi/keygen/parser/AbstractHoodieDateTimeParser.java rename to hudi-client/hudi-client-common/src/main/java/org/apache/hudi/keygen/parser/BaseHoodieDateTimeParser.java index 6fb05c30b..d45f97447 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/keygen/parser/AbstractHoodieDateTimeParser.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/keygen/parser/BaseHoodieDateTimeParser.java @@ -25,12 +25,12 @@ import org.joda.time.format.DateTimeFormatter; import java.io.Serializable; -public abstract class AbstractHoodieDateTimeParser implements Serializable { +public abstract class BaseHoodieDateTimeParser implements Serializable { protected final TypedProperties config; protected final String configInputDateFormatDelimiter; - public AbstractHoodieDateTimeParser(TypedProperties config) { + public BaseHoodieDateTimeParser(TypedProperties config) { this.config = config; this.configInputDateFormatDelimiter = initInputDateFormatDelimiter(); } diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/keygen/parser/HoodieDateTimeParserImpl.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/keygen/parser/HoodieDateTimeParser.java similarity index 96% rename from hudi-client/hudi-client-common/src/main/java/org/apache/hudi/keygen/parser/HoodieDateTimeParserImpl.java rename to hudi-client/hudi-client-common/src/main/java/org/apache/hudi/keygen/parser/HoodieDateTimeParser.java index 81960ea16..64f79f0e6 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/keygen/parser/HoodieDateTimeParserImpl.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/keygen/parser/HoodieDateTimeParser.java @@ -32,7 +32,7 @@ import java.util.Arrays; import java.util.Collections; import java.util.TimeZone; -public class HoodieDateTimeParserImpl extends AbstractHoodieDateTimeParser { +public class HoodieDateTimeParser extends BaseHoodieDateTimeParser { private String configInputDateFormatList; @@ -40,7 +40,7 @@ public class HoodieDateTimeParserImpl extends AbstractHoodieDateTimeParser { // https://docs.oracle.com/javase/8/docs/api/java/util/TimeZone.html private final DateTimeZone inputDateTimeZone; - public HoodieDateTimeParserImpl(TypedProperties config) { + public HoodieDateTimeParser(TypedProperties config) { super(config); KeyGenUtils.checkRequiredProperties(config, Arrays.asList(Config.TIMESTAMP_TYPE_FIELD_PROP, Config.TIMESTAMP_OUTPUT_DATE_FORMAT_PROP)); this.inputDateTimeZone = getInputDateTimeZone(); diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java index 841c48beb..49e2c8f74 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java @@ -24,7 +24,7 @@ import org.apache.hudi.avro.model.HoodieInstantInfo; import org.apache.hudi.avro.model.HoodieMetadataRecord; import org.apache.hudi.avro.model.HoodieRestoreMetadata; import org.apache.hudi.avro.model.HoodieRollbackMetadata; -import org.apache.hudi.client.AbstractHoodieWriteClient; +import org.apache.hudi.client.BaseHoodieWriteClient; import org.apache.hudi.common.config.HoodieMetadataConfig; import org.apache.hudi.common.config.SerializableConfiguration; import org.apache.hudi.common.data.HoodieData; @@ -682,7 +682,7 @@ public abstract class HoodieBackedTableMetadataWriter implements HoodieTableMeta * 2. In multi-writer scenario, a parallel operation with a greater instantTime may have completed creating a * deltacommit. */ - protected void compactIfNecessary(AbstractHoodieWriteClient writeClient, String instantTime) { + protected void compactIfNecessary(BaseHoodieWriteClient writeClient, String instantTime) { // finish off any pending compactions if any from previous attempt. writeClient.runAnyPendingCompactions(); @@ -706,7 +706,7 @@ public abstract class HoodieBackedTableMetadataWriter implements HoodieTableMeta } } - protected void cleanIfNecessary(AbstractHoodieWriteClient writeClient, String instantTime) { + protected void cleanIfNecessary(BaseHoodieWriteClient writeClient, String instantTime) { Option lastCompletedCompactionInstant = metadataMetaClient.reloadActiveTimeline() .getCommitTimeline().filterCompletedInstants().lastInstant(); if (lastCompletedCompactionInstant.isPresent() diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metrics/MetricsReporterFactory.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metrics/MetricsReporterFactory.java index dc9e80431..d81e337b2 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metrics/MetricsReporterFactory.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metrics/MetricsReporterFactory.java @@ -23,12 +23,12 @@ import org.apache.hudi.common.util.StringUtils; import org.apache.hudi.config.HoodieWriteConfig; import org.apache.hudi.exception.HoodieException; import org.apache.hudi.metrics.cloudwatch.CloudWatchMetricsReporter; +import org.apache.hudi.metrics.custom.CustomizableMetricsReporter; import org.apache.hudi.metrics.datadog.DatadogMetricsReporter; - -import com.codahale.metrics.MetricRegistry; import org.apache.hudi.metrics.prometheus.PrometheusReporter; import org.apache.hudi.metrics.prometheus.PushGatewayMetricsReporter; -import org.apache.hudi.metrics.userdefined.AbstractUserDefinedMetricsReporter; + +import com.codahale.metrics.MetricRegistry; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; @@ -47,9 +47,9 @@ public class MetricsReporterFactory { if (!StringUtils.isNullOrEmpty(reporterClassName)) { Object instance = ReflectionUtils.loadClass( reporterClassName, new Class[] {Properties.class, MetricRegistry.class}, config.getProps(), registry); - if (!(instance instanceof AbstractUserDefinedMetricsReporter)) { + if (!(instance instanceof CustomizableMetricsReporter)) { throw new HoodieException(config.getMetricReporterClassName() - + " is not a subclass of AbstractUserDefinedMetricsReporter"); + + " is not a subclass of CustomizableMetricsReporter"); } return (MetricsReporter) instance; } diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metrics/custom/CustomizableMetricsReporter.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metrics/custom/CustomizableMetricsReporter.java new file mode 100644 index 000000000..13574b1e1 --- /dev/null +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metrics/custom/CustomizableMetricsReporter.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.metrics.custom; + +import org.apache.hudi.metrics.MetricsReporter; + +import com.codahale.metrics.MetricRegistry; + +import java.util.Properties; + +/** + * Extensible metrics reporter for custom implementation. + */ +public abstract class CustomizableMetricsReporter extends MetricsReporter { + private Properties props; + private MetricRegistry registry; + + public CustomizableMetricsReporter(Properties props, MetricRegistry registry) { + this.props = props; + this.registry = registry; + } + + public Properties getProps() { + return props; + } + + public MetricRegistry getRegistry() { + return registry; + } +} diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metrics/userdefined/AbstractUserDefinedMetricsReporter.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metrics/userdefined/AbstractUserDefinedMetricsReporter.java index 0a0d7bbe1..715b9564c 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metrics/userdefined/AbstractUserDefinedMetricsReporter.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metrics/userdefined/AbstractUserDefinedMetricsReporter.java @@ -7,38 +7,31 @@ * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package org.apache.hudi.metrics.userdefined; +import org.apache.hudi.metrics.custom.CustomizableMetricsReporter; + import com.codahale.metrics.MetricRegistry; -import org.apache.hudi.metrics.MetricsReporter; + import java.util.Properties; /** - * Abstract class of user defined metrics reporter. + * @deprecated Extend {@link CustomizableMetricsReporter} instead. */ -public abstract class AbstractUserDefinedMetricsReporter extends MetricsReporter { - private Properties props; - private MetricRegistry registry; +@Deprecated +public abstract class AbstractUserDefinedMetricsReporter extends CustomizableMetricsReporter { public AbstractUserDefinedMetricsReporter(Properties props, MetricRegistry registry) { - this.props = props; - this.registry = registry; + super(props, registry); } - - public Properties getProps() { - return props; - } - - public MetricRegistry getRegistry() { - return registry; - } -} \ No newline at end of file +} diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/commit/AbstractBulkInsertHelper.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/commit/BaseBulkInsertHelper.java similarity index 96% rename from hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/commit/AbstractBulkInsertHelper.java rename to hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/commit/BaseBulkInsertHelper.java index 6e1ddeb72..29a13509a 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/commit/AbstractBulkInsertHelper.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/commit/BaseBulkInsertHelper.java @@ -26,7 +26,7 @@ import org.apache.hudi.table.BulkInsertPartitioner; import org.apache.hudi.table.HoodieTable; import org.apache.hudi.table.action.HoodieWriteMetadata; -public abstract class AbstractBulkInsertHelper { +public abstract class BaseBulkInsertHelper { /** * Mark instant as inflight, write input records, update index and return result. diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/commit/BaseCommitActionExecutor.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/commit/BaseCommitActionExecutor.java index 432aaf126..351f95ef5 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/commit/BaseCommitActionExecutor.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/commit/BaseCommitActionExecutor.java @@ -72,7 +72,7 @@ public abstract class BaseCommitActionExecutor */ -public abstract class AbstractDeleteHelper { +public abstract class BaseDeleteHelper { /** * Deduplicate Hoodie records, using the given deduplication function. diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/commit/AbstractMergeHelper.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/commit/BaseMergeHelper.java similarity index 98% rename from hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/commit/AbstractMergeHelper.java rename to hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/commit/BaseMergeHelper.java index 59a3323bc..5ead34814 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/commit/AbstractMergeHelper.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/commit/BaseMergeHelper.java @@ -47,7 +47,7 @@ import java.util.Iterator; /** * Helper to read records from previous version of base file and run Merge. */ -public abstract class AbstractMergeHelper { +public abstract class BaseMergeHelper { /** * Read records from previous version of base file and merge. diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/commit/AbstractWriteHelper.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/commit/BaseWriteHelper.java similarity index 97% rename from hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/commit/AbstractWriteHelper.java rename to hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/commit/BaseWriteHelper.java index 3f241944c..753bda97e 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/commit/AbstractWriteHelper.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/commit/BaseWriteHelper.java @@ -30,7 +30,7 @@ import org.apache.hudi.table.action.HoodieWriteMetadata; import java.time.Duration; import java.time.Instant; -public abstract class AbstractWriteHelper { +public abstract class BaseWriteHelper { public HoodieWriteMetadata write(String instantTime, I inputRecords, diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/DowngradeHandler.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/DowngradeHandler.java index 24b9d6f5d..45bbd78c3 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/DowngradeHandler.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/DowngradeHandler.java @@ -35,10 +35,10 @@ public interface DowngradeHandler { * @param config instance of {@link HoodieWriteConfig} to be used. * @param context instance of {@link HoodieEngineContext} to be used. * @param instantTime current instant time that should not touched. - * @param upgradeDowngradeHelper instance of {@link BaseUpgradeDowngradeHelper} to be used. + * @param upgradeDowngradeHelper instance of {@link SupportsUpgradeDowngrade} to be used. * @return Map of config properties and its values to be added to table properties. */ Map downgrade( HoodieWriteConfig config, HoodieEngineContext context, String instantTime, - BaseUpgradeDowngradeHelper upgradeDowngradeHelper); + SupportsUpgradeDowngrade upgradeDowngradeHelper); } diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/OneToTwoUpgradeHandler.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/OneToTwoUpgradeHandler.java index efa0fe472..dbf4d6159 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/OneToTwoUpgradeHandler.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/OneToTwoUpgradeHandler.java @@ -35,7 +35,7 @@ public class OneToTwoUpgradeHandler implements UpgradeHandler { @Override public Map upgrade( HoodieWriteConfig config, HoodieEngineContext context, String instantTime, - BaseUpgradeDowngradeHelper upgradeDowngradeHelper) { + SupportsUpgradeDowngrade upgradeDowngradeHelper) { Map tablePropsToAdd = new Hashtable<>(); tablePropsToAdd.put(HoodieTableConfig.PARTITION_FIELDS, upgradeDowngradeHelper.getPartitionColumns(config)); tablePropsToAdd.put(HoodieTableConfig.RECORDKEY_FIELDS, config.getString(KeyGeneratorOptions.RECORDKEY_FIELD_NAME.key())); diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/OneToZeroDowngradeHandler.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/OneToZeroDowngradeHandler.java index e6051cf32..14fe8e2b8 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/OneToZeroDowngradeHandler.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/OneToZeroDowngradeHandler.java @@ -40,7 +40,7 @@ public class OneToZeroDowngradeHandler implements DowngradeHandler { @Override public Map downgrade( HoodieWriteConfig config, HoodieEngineContext context, String instantTime, - BaseUpgradeDowngradeHelper upgradeDowngradeHelper) { + SupportsUpgradeDowngrade upgradeDowngradeHelper) { HoodieTable table = upgradeDowngradeHelper.getTable(config, context); // fetch pending commit info HoodieTimeline inflightTimeline = table.getMetaClient().getCommitsTimeline().filterPendingExcludingCompaction(); diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/BaseUpgradeDowngradeHelper.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/SupportsUpgradeDowngrade.java similarity index 97% rename from hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/BaseUpgradeDowngradeHelper.java rename to hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/SupportsUpgradeDowngrade.java index d3f157be9..5e6b9db91 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/BaseUpgradeDowngradeHelper.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/SupportsUpgradeDowngrade.java @@ -26,7 +26,7 @@ import org.apache.hudi.table.HoodieTable; /** * Interface for engine-specific logic needed for upgrade and downgrade actions. */ -public interface BaseUpgradeDowngradeHelper { +public interface SupportsUpgradeDowngrade { /** * @param config Write config. * @param context {@link HoodieEngineContext} instance to use. diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/ThreeToTwoDowngradeHandler.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/ThreeToTwoDowngradeHandler.java index 964859c0a..4f209f05f 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/ThreeToTwoDowngradeHandler.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/ThreeToTwoDowngradeHandler.java @@ -33,7 +33,7 @@ import java.util.Map; public class ThreeToTwoDowngradeHandler implements DowngradeHandler { @Override - public Map downgrade(HoodieWriteConfig config, HoodieEngineContext context, String instantTime, BaseUpgradeDowngradeHelper upgradeDowngradeHelper) { + public Map downgrade(HoodieWriteConfig config, HoodieEngineContext context, String instantTime, SupportsUpgradeDowngrade upgradeDowngradeHelper) { if (config.isMetadataTableEnabled()) { // Metadata Table in version 3 is synchronous and in version 2 is asynchronous. Downgrading to asynchronous // removes the checks in code to decide whether to use a LogBlock or not. Also, the schema for the diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/TwoToOneDowngradeHandler.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/TwoToOneDowngradeHandler.java index ee638a16f..0a2e0f813 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/TwoToOneDowngradeHandler.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/TwoToOneDowngradeHandler.java @@ -54,7 +54,7 @@ public class TwoToOneDowngradeHandler implements DowngradeHandler { @Override public Map downgrade( HoodieWriteConfig config, HoodieEngineContext context, String instantTime, - BaseUpgradeDowngradeHelper upgradeDowngradeHelper) { + SupportsUpgradeDowngrade upgradeDowngradeHelper) { HoodieTable table = upgradeDowngradeHelper.getTable(config, context); HoodieTableMetaClient metaClient = table.getMetaClient(); diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/TwoToThreeUpgradeHandler.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/TwoToThreeUpgradeHandler.java index bff3788d5..c13d21ec2 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/TwoToThreeUpgradeHandler.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/TwoToThreeUpgradeHandler.java @@ -35,7 +35,7 @@ import java.util.Map; */ public class TwoToThreeUpgradeHandler implements UpgradeHandler { @Override - public Map upgrade(HoodieWriteConfig config, HoodieEngineContext context, String instantTime, BaseUpgradeDowngradeHelper upgradeDowngradeHelper) { + public Map upgrade(HoodieWriteConfig config, HoodieEngineContext context, String instantTime, SupportsUpgradeDowngrade upgradeDowngradeHelper) { if (config.isMetadataTableEnabled()) { // Metadata Table in version 2 is asynchronous and in version 3 is synchronous. Synchronous table will not // sync any instants not already synced. So its simpler to re-bootstrap the table. Also, the schema for the diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/UpgradeDowngrade.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/UpgradeDowngrade.java index 0e8f752a8..e1e7e2b33 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/UpgradeDowngrade.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/UpgradeDowngrade.java @@ -42,7 +42,7 @@ public class UpgradeDowngrade { private static final Logger LOG = LogManager.getLogger(UpgradeDowngrade.class); public static final String HOODIE_UPDATED_PROPERTY_FILE = "hoodie.properties.updated"; - private final BaseUpgradeDowngradeHelper upgradeDowngradeHelper; + private final SupportsUpgradeDowngrade upgradeDowngradeHelper; private HoodieTableMetaClient metaClient; protected HoodieWriteConfig config; protected HoodieEngineContext context; @@ -52,7 +52,7 @@ public class UpgradeDowngrade { public UpgradeDowngrade( HoodieTableMetaClient metaClient, HoodieWriteConfig config, HoodieEngineContext context, - BaseUpgradeDowngradeHelper upgradeDowngradeHelper) { + SupportsUpgradeDowngrade upgradeDowngradeHelper) { this.metaClient = metaClient; this.config = config; this.context = context; diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/UpgradeHandler.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/UpgradeHandler.java index 9dc477ffc..147aa4d8a 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/UpgradeHandler.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/UpgradeHandler.java @@ -35,10 +35,10 @@ public interface UpgradeHandler { * @param config instance of {@link HoodieWriteConfig} to be used. * @param context instance of {@link HoodieEngineContext} to be used. * @param instantTime current instant time that should not be touched. - * @param upgradeDowngradeHelper instance of {@link BaseUpgradeDowngradeHelper} to be used. + * @param upgradeDowngradeHelper instance of {@link SupportsUpgradeDowngrade} to be used. * @return Map of config properties and its values to be added to table properties. */ Map upgrade( HoodieWriteConfig config, HoodieEngineContext context, String instantTime, - BaseUpgradeDowngradeHelper upgradeDowngradeHelper); + SupportsUpgradeDowngrade upgradeDowngradeHelper); } diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/ZeroToOneUpgradeHandler.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/ZeroToOneUpgradeHandler.java index 1aebbf6b4..6a114154c 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/ZeroToOneUpgradeHandler.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/ZeroToOneUpgradeHandler.java @@ -57,7 +57,7 @@ public class ZeroToOneUpgradeHandler implements UpgradeHandler { @Override public Map upgrade( HoodieWriteConfig config, HoodieEngineContext context, String instantTime, - BaseUpgradeDowngradeHelper upgradeDowngradeHelper) { + SupportsUpgradeDowngrade upgradeDowngradeHelper) { // fetch pending commit info HoodieTable table = upgradeDowngradeHelper.getTable(config, context); HoodieTimeline inflightTimeline = table.getMetaClient().getCommitsTimeline().filterPendingExcludingCompaction(); diff --git a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metrics/TestMetricsReporterFactory.java b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metrics/TestMetricsReporterFactory.java index edd2302a6..3689755e4 100644 --- a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metrics/TestMetricsReporterFactory.java +++ b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metrics/TestMetricsReporterFactory.java @@ -21,10 +21,10 @@ package org.apache.hudi.metrics; import org.apache.hudi.common.config.TypedProperties; import org.apache.hudi.config.HoodieWriteConfig; +import org.apache.hudi.exception.HoodieException; +import org.apache.hudi.metrics.custom.CustomizableMetricsReporter; import com.codahale.metrics.MetricRegistry; -import org.apache.hudi.exception.HoodieException; -import org.apache.hudi.metrics.userdefined.AbstractUserDefinedMetricsReporter; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; @@ -63,7 +63,7 @@ public class TestMetricsReporterFactory { when(config.getProps()).thenReturn(props); MetricsReporter reporter = MetricsReporterFactory.createReporter(config, registry); - assertTrue(reporter instanceof AbstractUserDefinedMetricsReporter); + assertTrue(reporter instanceof CustomizableMetricsReporter); assertEquals(props, ((DummyMetricsReporter) reporter).getProps()); assertEquals(registry, ((DummyMetricsReporter) reporter).getRegistry()); } @@ -75,7 +75,7 @@ public class TestMetricsReporterFactory { assertThrows(HoodieException.class, () -> MetricsReporterFactory.createReporter(config, registry)); } - public static class DummyMetricsReporter extends AbstractUserDefinedMetricsReporter { + public static class DummyMetricsReporter extends CustomizableMetricsReporter { public DummyMetricsReporter(Properties props, MetricRegistry registry) { super(props, registry); diff --git a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/testutils/providers/HoodieWriteClientProvider.java b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/testutils/providers/HoodieWriteClientProvider.java index 9bc559deb..f67e158c8 100644 --- a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/testutils/providers/HoodieWriteClientProvider.java +++ b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/testutils/providers/HoodieWriteClientProvider.java @@ -19,12 +19,12 @@ package org.apache.hudi.testutils.providers; -import org.apache.hudi.client.AbstractHoodieWriteClient; +import org.apache.hudi.client.BaseHoodieWriteClient; import org.apache.hudi.config.HoodieWriteConfig; import java.io.IOException; public interface HoodieWriteClientProvider { - AbstractHoodieWriteClient getHoodieWriteClient(HoodieWriteConfig cfg) throws IOException; + BaseHoodieWriteClient getHoodieWriteClient(HoodieWriteConfig cfg) throws IOException; } diff --git a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/client/HoodieFlinkWriteClient.java b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/client/HoodieFlinkWriteClient.java index 4108ba425..50477c3f8 100644 --- a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/client/HoodieFlinkWriteClient.java +++ b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/client/HoodieFlinkWriteClient.java @@ -78,7 +78,7 @@ import java.util.stream.Collectors; @SuppressWarnings("checkstyle:LineLength") public class HoodieFlinkWriteClient extends - AbstractHoodieWriteClient>, List, List> { + BaseHoodieWriteClient>, List, List> { private static final Logger LOG = LoggerFactory.getLogger(HoodieFlinkWriteClient.class); diff --git a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/metadata/FlinkHoodieBackedTableMetadataWriter.java b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/metadata/FlinkHoodieBackedTableMetadataWriter.java index af9fee068..f4ce686d2 100644 --- a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/metadata/FlinkHoodieBackedTableMetadataWriter.java +++ b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/metadata/FlinkHoodieBackedTableMetadataWriter.java @@ -132,7 +132,7 @@ public class FlinkHoodieBackedTableMetadataWriter extends HoodieBackedTableMetad throw new HoodieMetadataException("Failed to commit metadata table records at instant " + instantTime); } }); - // flink does not support auto-commit yet, also the auto commit logic is not complete as AbstractHoodieWriteClient now. + // flink does not support auto-commit yet, also the auto commit logic is not complete as BaseHoodieWriteClient now. writeClient.commit(instantTime, statuses, Option.empty(), HoodieActiveTimeline.DELTA_COMMIT_ACTION, Collections.emptyMap()); // reload timeline diff --git a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/table/action/commit/FlinkDeleteHelper.java b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/table/action/commit/FlinkDeleteHelper.java index 05ac93725..8df050078 100644 --- a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/table/action/commit/FlinkDeleteHelper.java +++ b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/table/action/commit/FlinkDeleteHelper.java @@ -43,7 +43,7 @@ import java.util.stream.Collectors; @SuppressWarnings("checkstyle:LineLength") public class FlinkDeleteHelper extends - AbstractDeleteHelper>, List, List, R> { + BaseDeleteHelper>, List, List, R> { private FlinkDeleteHelper() { } diff --git a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/table/action/commit/FlinkMergeHelper.java b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/table/action/commit/FlinkMergeHelper.java index 5ed6d5d52..c88b4ee66 100644 --- a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/table/action/commit/FlinkMergeHelper.java +++ b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/table/action/commit/FlinkMergeHelper.java @@ -45,7 +45,7 @@ import java.util.Iterator; import scala.collection.immutable.List; -public class FlinkMergeHelper extends AbstractMergeHelper>, +public class FlinkMergeHelper extends BaseMergeHelper>, List, List> { private FlinkMergeHelper() { diff --git a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/table/action/commit/FlinkWriteHelper.java b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/table/action/commit/FlinkWriteHelper.java index 3914e486f..3c689539f 100644 --- a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/table/action/commit/FlinkWriteHelper.java +++ b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/table/action/commit/FlinkWriteHelper.java @@ -48,7 +48,7 @@ import java.util.stream.Collectors; *

Computing the records batch locations all at a time is a pressure to the engine, * we should avoid that in streaming system. */ -public class FlinkWriteHelper extends AbstractWriteHelper>, +public class FlinkWriteHelper extends BaseWriteHelper>, List, List, R> { private FlinkWriteHelper() { diff --git a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/table/upgrade/FlinkUpgradeDowngradeHelper.java b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/table/upgrade/FlinkUpgradeDowngradeHelper.java index d097d2e60..69acce562 100644 --- a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/table/upgrade/FlinkUpgradeDowngradeHelper.java +++ b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/table/upgrade/FlinkUpgradeDowngradeHelper.java @@ -29,7 +29,7 @@ import org.apache.hudi.table.HoodieTable; /** * Flink upgrade and downgrade helper. */ -public class FlinkUpgradeDowngradeHelper implements BaseUpgradeDowngradeHelper { +public class FlinkUpgradeDowngradeHelper implements SupportsUpgradeDowngrade { private static final FlinkUpgradeDowngradeHelper SINGLETON_INSTANCE = new FlinkUpgradeDowngradeHelper(); diff --git a/hudi-client/hudi-java-client/src/main/java/org/apache/hudi/client/HoodieJavaWriteClient.java b/hudi-client/hudi-java-client/src/main/java/org/apache/hudi/client/HoodieJavaWriteClient.java index 212187b2d..7af24c8b5 100644 --- a/hudi-client/hudi-java-client/src/main/java/org/apache/hudi/client/HoodieJavaWriteClient.java +++ b/hudi-client/hudi-java-client/src/main/java/org/apache/hudi/client/HoodieJavaWriteClient.java @@ -50,7 +50,7 @@ import java.util.Map; import java.util.stream.Collectors; public class HoodieJavaWriteClient extends - AbstractHoodieWriteClient>, List, List> { + BaseHoodieWriteClient>, List, List> { public HoodieJavaWriteClient(HoodieEngineContext context, HoodieWriteConfig clientConfig) { super(context, clientConfig); diff --git a/hudi-client/hudi-java-client/src/main/java/org/apache/hudi/table/action/commit/JavaBulkInsertHelper.java b/hudi-client/hudi-java-client/src/main/java/org/apache/hudi/table/action/commit/JavaBulkInsertHelper.java index cdfa303cd..87b7fe74e 100644 --- a/hudi-client/hudi-java-client/src/main/java/org/apache/hudi/table/action/commit/JavaBulkInsertHelper.java +++ b/hudi-client/hudi-java-client/src/main/java/org/apache/hudi/table/action/commit/JavaBulkInsertHelper.java @@ -39,12 +39,12 @@ import java.util.ArrayList; import java.util.List; /** - * A java implementation of {@link AbstractBulkInsertHelper}. + * A java implementation of {@link BaseBulkInsertHelper}. * * @param */ @SuppressWarnings("checkstyle:LineLength") -public class JavaBulkInsertHelper extends AbstractBulkInsertHelper>, +public class JavaBulkInsertHelper extends BaseBulkInsertHelper>, List, List, R> { private JavaBulkInsertHelper() { diff --git a/hudi-client/hudi-java-client/src/main/java/org/apache/hudi/table/action/commit/JavaDeleteHelper.java b/hudi-client/hudi-java-client/src/main/java/org/apache/hudi/table/action/commit/JavaDeleteHelper.java index fc81b787f..c75d829cf 100644 --- a/hudi-client/hudi-java-client/src/main/java/org/apache/hudi/table/action/commit/JavaDeleteHelper.java +++ b/hudi-client/hudi-java-client/src/main/java/org/apache/hudi/table/action/commit/JavaDeleteHelper.java @@ -43,7 +43,7 @@ import java.util.stream.Collectors; @SuppressWarnings("checkstyle:LineLength") public class JavaDeleteHelper extends - AbstractDeleteHelper>, List, List, R> { + BaseDeleteHelper>, List, List, R> { private JavaDeleteHelper() { } diff --git a/hudi-client/hudi-java-client/src/main/java/org/apache/hudi/table/action/commit/JavaMergeHelper.java b/hudi-client/hudi-java-client/src/main/java/org/apache/hudi/table/action/commit/JavaMergeHelper.java index a55121472..0df6d3a90 100644 --- a/hudi-client/hudi-java-client/src/main/java/org/apache/hudi/table/action/commit/JavaMergeHelper.java +++ b/hudi-client/hudi-java-client/src/main/java/org/apache/hudi/table/action/commit/JavaMergeHelper.java @@ -44,7 +44,7 @@ import java.io.IOException; import java.util.Iterator; import java.util.List; -public class JavaMergeHelper extends AbstractMergeHelper>, +public class JavaMergeHelper extends BaseMergeHelper>, List, List> { private JavaMergeHelper() { diff --git a/hudi-client/hudi-java-client/src/main/java/org/apache/hudi/table/action/commit/JavaWriteHelper.java b/hudi-client/hudi-java-client/src/main/java/org/apache/hudi/table/action/commit/JavaWriteHelper.java index 8af7707ea..6d420e373 100644 --- a/hudi-client/hudi-java-client/src/main/java/org/apache/hudi/table/action/commit/JavaWriteHelper.java +++ b/hudi-client/hudi-java-client/src/main/java/org/apache/hudi/table/action/commit/JavaWriteHelper.java @@ -33,7 +33,7 @@ import java.util.Map; import java.util.Objects; import java.util.stream.Collectors; -public class JavaWriteHelper extends AbstractWriteHelper>, +public class JavaWriteHelper extends BaseWriteHelper>, List, List, R> { private JavaWriteHelper() { diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/async/SparkAsyncClusteringService.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/async/SparkAsyncClusteringService.java index ce436ba03..8f6535b11 100644 --- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/async/SparkAsyncClusteringService.java +++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/async/SparkAsyncClusteringService.java @@ -19,8 +19,8 @@ package org.apache.hudi.async; -import org.apache.hudi.client.AbstractClusteringClient; -import org.apache.hudi.client.AbstractHoodieWriteClient; +import org.apache.hudi.client.BaseClusterer; +import org.apache.hudi.client.BaseHoodieWriteClient; import org.apache.hudi.client.HoodieSparkClusteringClient; /** @@ -28,12 +28,12 @@ import org.apache.hudi.client.HoodieSparkClusteringClient; */ public class SparkAsyncClusteringService extends AsyncClusteringService { - public SparkAsyncClusteringService(AbstractHoodieWriteClient writeClient) { + public SparkAsyncClusteringService(BaseHoodieWriteClient writeClient) { super(writeClient); } @Override - protected AbstractClusteringClient createClusteringClient(AbstractHoodieWriteClient client) { + protected BaseClusterer createClusteringClient(BaseHoodieWriteClient client) { return new HoodieSparkClusteringClient(client); } } diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/async/SparkAsyncCompactService.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/async/SparkAsyncCompactService.java index 5235a3cd0..d54fe386b 100644 --- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/async/SparkAsyncCompactService.java +++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/async/SparkAsyncCompactService.java @@ -18,19 +18,19 @@ package org.apache.hudi.async; -import org.apache.hudi.client.AbstractCompactor; -import org.apache.hudi.client.AbstractHoodieWriteClient; +import org.apache.hudi.client.BaseCompactor; +import org.apache.hudi.client.BaseHoodieWriteClient; import org.apache.hudi.client.HoodieSparkCompactor; import org.apache.hudi.common.engine.HoodieEngineContext; public class SparkAsyncCompactService extends AsyncCompactService { - public SparkAsyncCompactService(HoodieEngineContext context, AbstractHoodieWriteClient client) { + public SparkAsyncCompactService(HoodieEngineContext context, BaseHoodieWriteClient client) { super(context, client); } @Override - protected AbstractCompactor createCompactor(AbstractHoodieWriteClient client) { + protected BaseCompactor createCompactor(BaseHoodieWriteClient client) { return new HoodieSparkCompactor(client, this.context); } } diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/HoodieSparkClusteringClient.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/HoodieSparkClusteringClient.java index 16e54a215..0812b366a 100644 --- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/HoodieSparkClusteringClient.java +++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/HoodieSparkClusteringClient.java @@ -38,12 +38,12 @@ import java.util.stream.Stream; * Async clustering client for Spark datasource. */ public class HoodieSparkClusteringClient extends - AbstractClusteringClient>, JavaRDD, JavaRDD> { + BaseClusterer>, JavaRDD, JavaRDD> { private static final Logger LOG = LogManager.getLogger(HoodieSparkClusteringClient.class); public HoodieSparkClusteringClient( - AbstractHoodieWriteClient>, JavaRDD, JavaRDD> clusteringClient) { + BaseHoodieWriteClient>, JavaRDD, JavaRDD> clusteringClient) { super(clusteringClient); } diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/HoodieSparkCompactor.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/HoodieSparkCompactor.java index 60a064ab3..ca5684a19 100644 --- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/HoodieSparkCompactor.java +++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/HoodieSparkCompactor.java @@ -31,12 +31,12 @@ import org.apache.spark.api.java.JavaRDD; import java.io.IOException; -public class HoodieSparkCompactor extends AbstractCompactor extends BaseCompactor>, JavaRDD, JavaRDD> { private static final Logger LOG = LogManager.getLogger(HoodieSparkCompactor.class); private transient HoodieEngineContext context; - public HoodieSparkCompactor(AbstractHoodieWriteClient>, JavaRDD, JavaRDD> compactionClient, + public HoodieSparkCompactor(BaseHoodieWriteClient>, JavaRDD, JavaRDD> compactionClient, HoodieEngineContext context) { super(compactionClient); this.context = context; diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/SparkRDDWriteClient.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/SparkRDDWriteClient.java index f4cecbeaa..f541720fd 100644 --- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/SparkRDDWriteClient.java +++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/SparkRDDWriteClient.java @@ -74,7 +74,7 @@ import java.util.stream.Collectors; @SuppressWarnings("checkstyle:LineLength") public class SparkRDDWriteClient extends - AbstractHoodieWriteClient>, JavaRDD, JavaRDD> { + BaseHoodieWriteClient>, JavaRDD, JavaRDD> { private static final Logger LOG = LogManager.getLogger(SparkRDDWriteClient.class); diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/commit/SparkBulkInsertHelper.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/commit/SparkBulkInsertHelper.java index 4644d29e0..d17b9b45b 100644 --- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/commit/SparkBulkInsertHelper.java +++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/commit/SparkBulkInsertHelper.java @@ -41,12 +41,12 @@ import java.util.stream.Collectors; import java.util.stream.IntStream; /** - * A spark implementation of {@link AbstractBulkInsertHelper}. + * A spark implementation of {@link BaseBulkInsertHelper}. * * @param */ @SuppressWarnings("checkstyle:LineLength") -public class SparkBulkInsertHelper extends AbstractBulkInsertHelper>, +public class SparkBulkInsertHelper extends BaseBulkInsertHelper>, JavaRDD, JavaRDD, R> { private SparkBulkInsertHelper() { diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/commit/SparkDeleteHelper.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/commit/SparkDeleteHelper.java index 5c3b4ca22..bae3556df 100644 --- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/commit/SparkDeleteHelper.java +++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/commit/SparkDeleteHelper.java @@ -42,13 +42,13 @@ import java.time.Instant; import java.util.HashMap; /** - * A spark implementation of {@link AbstractDeleteHelper}. + * A spark implementation of {@link BaseDeleteHelper}. * * @param */ @SuppressWarnings("checkstyle:LineLength") public class SparkDeleteHelper extends - AbstractDeleteHelper>, JavaRDD, JavaRDD, R> { + BaseDeleteHelper>, JavaRDD, JavaRDD, R> { private SparkDeleteHelper() { } diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/commit/SparkMergeHelper.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/commit/SparkMergeHelper.java index 5e82dbd8c..806154513 100644 --- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/commit/SparkMergeHelper.java +++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/commit/SparkMergeHelper.java @@ -43,7 +43,7 @@ import org.apache.spark.api.java.JavaRDD; import java.io.IOException; import java.util.Iterator; -public class SparkMergeHelper extends AbstractMergeHelper>, +public class SparkMergeHelper extends BaseMergeHelper>, JavaRDD, JavaRDD> { private SparkMergeHelper() { diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/commit/SparkWriteHelper.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/commit/SparkWriteHelper.java index f4eff44a2..489526410 100644 --- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/commit/SparkWriteHelper.java +++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/commit/SparkWriteHelper.java @@ -32,11 +32,11 @@ import org.apache.spark.api.java.JavaRDD; import scala.Tuple2; /** - * A spark implementation of {@link AbstractWriteHelper}. + * A spark implementation of {@link BaseWriteHelper}. * * @param */ -public class SparkWriteHelper extends AbstractWriteHelper>, +public class SparkWriteHelper extends BaseWriteHelper>, JavaRDD, JavaRDD, R> { private SparkWriteHelper() { } diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/deltacommit/AbstractSparkDeltaCommitActionExecutor.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/deltacommit/BaseSparkDeltaCommitActionExecutor.java similarity index 90% rename from hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/deltacommit/AbstractSparkDeltaCommitActionExecutor.java rename to hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/deltacommit/BaseSparkDeltaCommitActionExecutor.java index 3b3edd308..222506e7b 100644 --- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/deltacommit/AbstractSparkDeltaCommitActionExecutor.java +++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/deltacommit/BaseSparkDeltaCommitActionExecutor.java @@ -43,19 +43,19 @@ import java.util.Collections; import java.util.Iterator; import java.util.List; -public abstract class AbstractSparkDeltaCommitActionExecutor> +public abstract class BaseSparkDeltaCommitActionExecutor> extends BaseSparkCommitActionExecutor { - private static final Logger LOG = LogManager.getLogger(AbstractSparkDeltaCommitActionExecutor.class); + private static final Logger LOG = LogManager.getLogger(BaseSparkDeltaCommitActionExecutor.class); // UpsertPartitioner for MergeOnRead table type private SparkUpsertDeltaCommitPartitioner mergeOnReadUpsertPartitioner; - public AbstractSparkDeltaCommitActionExecutor(HoodieSparkEngineContext context, HoodieWriteConfig config, HoodieTable table, + public BaseSparkDeltaCommitActionExecutor(HoodieSparkEngineContext context, HoodieWriteConfig config, HoodieTable table, String instantTime, WriteOperationType operationType) { this(context, config, table, instantTime, operationType, Option.empty()); } - public AbstractSparkDeltaCommitActionExecutor(HoodieSparkEngineContext context, HoodieWriteConfig config, HoodieTable table, + public BaseSparkDeltaCommitActionExecutor(HoodieSparkEngineContext context, HoodieWriteConfig config, HoodieTable table, String instantTime, WriteOperationType operationType, Option> extraMetadata) { super(context, config, table, instantTime, operationType, extraMetadata); diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/deltacommit/SparkBulkInsertDeltaCommitActionExecutor.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/deltacommit/SparkBulkInsertDeltaCommitActionExecutor.java index 281304d95..17baf1d4c 100644 --- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/deltacommit/SparkBulkInsertDeltaCommitActionExecutor.java +++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/deltacommit/SparkBulkInsertDeltaCommitActionExecutor.java @@ -18,8 +18,6 @@ package org.apache.hudi.table.action.deltacommit; -import java.util.Map; - import org.apache.hudi.client.WriteStatus; import org.apache.hudi.client.common.HoodieSparkEngineContext; import org.apache.hudi.common.model.HoodieRecord; @@ -28,15 +26,17 @@ import org.apache.hudi.common.model.WriteOperationType; import org.apache.hudi.common.util.Option; import org.apache.hudi.config.HoodieWriteConfig; import org.apache.hudi.exception.HoodieInsertException; -import org.apache.hudi.table.HoodieTable; import org.apache.hudi.table.BulkInsertPartitioner; - +import org.apache.hudi.table.HoodieTable; import org.apache.hudi.table.action.HoodieWriteMetadata; import org.apache.hudi.table.action.commit.SparkBulkInsertHelper; + import org.apache.spark.api.java.JavaRDD; +import java.util.Map; + public class SparkBulkInsertDeltaCommitActionExecutor> - extends AbstractSparkDeltaCommitActionExecutor { + extends BaseSparkDeltaCommitActionExecutor { private final JavaRDD> inputRecordsRDD; private final Option> bulkInsertPartitioner; diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/deltacommit/SparkBulkInsertPreppedDeltaCommitActionExecutor.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/deltacommit/SparkBulkInsertPreppedDeltaCommitActionExecutor.java index 21fc013af..a4d7493e8 100644 --- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/deltacommit/SparkBulkInsertPreppedDeltaCommitActionExecutor.java +++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/deltacommit/SparkBulkInsertPreppedDeltaCommitActionExecutor.java @@ -26,15 +26,15 @@ import org.apache.hudi.common.model.WriteOperationType; import org.apache.hudi.common.util.Option; import org.apache.hudi.config.HoodieWriteConfig; import org.apache.hudi.exception.HoodieInsertException; -import org.apache.hudi.table.HoodieTable; import org.apache.hudi.table.BulkInsertPartitioner; - +import org.apache.hudi.table.HoodieTable; import org.apache.hudi.table.action.HoodieWriteMetadata; import org.apache.hudi.table.action.commit.SparkBulkInsertHelper; + import org.apache.spark.api.java.JavaRDD; public class SparkBulkInsertPreppedDeltaCommitActionExecutor> - extends AbstractSparkDeltaCommitActionExecutor { + extends BaseSparkDeltaCommitActionExecutor { private final JavaRDD> preppedInputRecordRdd; private final Option> bulkInsertPartitioner; @@ -61,4 +61,4 @@ public class SparkBulkInsertPreppedDeltaCommitActionExecutor> - extends AbstractSparkDeltaCommitActionExecutor { + extends BaseSparkDeltaCommitActionExecutor { private final JavaRDD keys; diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/deltacommit/SparkInsertDeltaCommitActionExecutor.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/deltacommit/SparkInsertDeltaCommitActionExecutor.java index 7dd91710d..7e38823fc 100644 --- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/deltacommit/SparkInsertDeltaCommitActionExecutor.java +++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/deltacommit/SparkInsertDeltaCommitActionExecutor.java @@ -25,13 +25,13 @@ import org.apache.hudi.common.model.HoodieRecordPayload; import org.apache.hudi.common.model.WriteOperationType; import org.apache.hudi.config.HoodieWriteConfig; import org.apache.hudi.table.HoodieTable; - import org.apache.hudi.table.action.HoodieWriteMetadata; import org.apache.hudi.table.action.commit.SparkWriteHelper; + import org.apache.spark.api.java.JavaRDD; public class SparkInsertDeltaCommitActionExecutor> - extends AbstractSparkDeltaCommitActionExecutor { + extends BaseSparkDeltaCommitActionExecutor { private final JavaRDD> inputRecordsRDD; diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/deltacommit/SparkInsertPreppedDeltaCommitActionExecutor.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/deltacommit/SparkInsertPreppedDeltaCommitActionExecutor.java index 1f1e0165b..e401d9555 100644 --- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/deltacommit/SparkInsertPreppedDeltaCommitActionExecutor.java +++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/deltacommit/SparkInsertPreppedDeltaCommitActionExecutor.java @@ -26,10 +26,11 @@ import org.apache.hudi.common.model.WriteOperationType; import org.apache.hudi.config.HoodieWriteConfig; import org.apache.hudi.table.HoodieTable; import org.apache.hudi.table.action.HoodieWriteMetadata; + import org.apache.spark.api.java.JavaRDD; public class SparkInsertPreppedDeltaCommitActionExecutor> - extends AbstractSparkDeltaCommitActionExecutor { + extends BaseSparkDeltaCommitActionExecutor { private final JavaRDD> preppedRecords; diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/deltacommit/SparkUpsertDeltaCommitActionExecutor.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/deltacommit/SparkUpsertDeltaCommitActionExecutor.java index c6f3901a3..c63be6289 100644 --- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/deltacommit/SparkUpsertDeltaCommitActionExecutor.java +++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/deltacommit/SparkUpsertDeltaCommitActionExecutor.java @@ -24,13 +24,13 @@ import org.apache.hudi.common.model.HoodieRecordPayload; import org.apache.hudi.common.model.WriteOperationType; import org.apache.hudi.config.HoodieWriteConfig; import org.apache.hudi.table.HoodieTable; - import org.apache.hudi.table.action.HoodieWriteMetadata; import org.apache.hudi.table.action.commit.SparkWriteHelper; + import org.apache.spark.api.java.JavaRDD; public class SparkUpsertDeltaCommitActionExecutor> - extends AbstractSparkDeltaCommitActionExecutor { + extends BaseSparkDeltaCommitActionExecutor { private JavaRDD> inputRecordsRDD; diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/deltacommit/SparkUpsertPreppedDeltaCommitActionExecutor.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/deltacommit/SparkUpsertPreppedDeltaCommitActionExecutor.java index 3509efa6b..f593fea77 100644 --- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/deltacommit/SparkUpsertPreppedDeltaCommitActionExecutor.java +++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/deltacommit/SparkUpsertPreppedDeltaCommitActionExecutor.java @@ -26,10 +26,11 @@ import org.apache.hudi.common.model.WriteOperationType; import org.apache.hudi.config.HoodieWriteConfig; import org.apache.hudi.table.HoodieTable; import org.apache.hudi.table.action.HoodieWriteMetadata; + import org.apache.spark.api.java.JavaRDD; public class SparkUpsertPreppedDeltaCommitActionExecutor> - extends AbstractSparkDeltaCommitActionExecutor { + extends BaseSparkDeltaCommitActionExecutor { private final JavaRDD> preppedRecords; diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/upgrade/SparkUpgradeDowngradeHelper.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/upgrade/SparkUpgradeDowngradeHelper.java index f943b7017..1a911d5b4 100644 --- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/upgrade/SparkUpgradeDowngradeHelper.java +++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/upgrade/SparkUpgradeDowngradeHelper.java @@ -28,7 +28,7 @@ import org.apache.hudi.table.HoodieTable; /** * Spark upgrade and downgrade helper. */ -public class SparkUpgradeDowngradeHelper implements BaseUpgradeDowngradeHelper { +public class SparkUpgradeDowngradeHelper implements SupportsUpgradeDowngrade { private static final SparkUpgradeDowngradeHelper SINGLETON_INSTANCE = new SparkUpgradeDowngradeHelper(); diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieBackedMetadata.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieBackedMetadata.java index efea08b41..88a6f02fc 100644 --- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieBackedMetadata.java +++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieBackedMetadata.java @@ -18,7 +18,6 @@ package org.apache.hudi.client.functional; -import org.apache.avro.Schema; import org.apache.hudi.avro.HoodieAvroUtils; import org.apache.hudi.avro.model.HoodieMetadataRecord; import org.apache.hudi.client.SparkRDDWriteClient; @@ -90,6 +89,7 @@ import org.apache.hudi.table.upgrade.SparkUpgradeDowngradeHelper; import org.apache.hudi.table.upgrade.UpgradeDowngrade; import org.apache.hudi.testutils.MetadataMergeWriteStatus; +import org.apache.avro.Schema; import org.apache.avro.generic.GenericRecord; import org.apache.avro.generic.IndexedRecord; import org.apache.hadoop.fs.FSDataOutputStream; diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieClientOnCopyOnWriteStorage.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieClientOnCopyOnWriteStorage.java index 4466d4672..e97a0689a 100644 --- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieClientOnCopyOnWriteStorage.java +++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieClientOnCopyOnWriteStorage.java @@ -21,7 +21,7 @@ package org.apache.hudi.client.functional; import org.apache.hudi.avro.model.HoodieCleanMetadata; import org.apache.hudi.avro.model.HoodieClusteringPlan; import org.apache.hudi.avro.model.HoodieRequestedReplaceMetadata; -import org.apache.hudi.client.AbstractHoodieWriteClient; +import org.apache.hudi.client.BaseHoodieWriteClient; import org.apache.hudi.client.HoodieWriteResult; import org.apache.hudi.client.SparkRDDWriteClient; import org.apache.hudi.client.SparkTaskContextSupplier; @@ -687,7 +687,7 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase { } /** - * Test one of HoodieConcatHandle w/ {@link AbstractHoodieWriteClient#insert(Object, String)} API. + * Test one of HoodieConcatHandle w/ {@link BaseHoodieWriteClient#insert(Object, String)} API. * * @param config Write Config * @throws Exception in case of error diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/TestHoodieMergeOnReadTable.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/TestHoodieMergeOnReadTable.java index 8b8df197b..ac77126fb 100644 --- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/TestHoodieMergeOnReadTable.java +++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/TestHoodieMergeOnReadTable.java @@ -44,7 +44,7 @@ import org.apache.hudi.index.HoodieIndex; import org.apache.hudi.index.HoodieIndex.IndexType; import org.apache.hudi.metadata.HoodieTableMetadataWriter; import org.apache.hudi.metadata.SparkHoodieBackedTableMetadataWriter; -import org.apache.hudi.table.action.deltacommit.AbstractSparkDeltaCommitActionExecutor; +import org.apache.hudi.table.action.deltacommit.BaseSparkDeltaCommitActionExecutor; import org.apache.hudi.table.action.deltacommit.SparkDeleteDeltaCommitActionExecutor; import org.apache.hudi.testutils.HoodieClientTestUtils; import org.apache.hudi.testutils.HoodieMergeOnReadTestUtils; @@ -552,7 +552,7 @@ public class TestHoodieMergeOnReadTable extends SparkClientFunctionalTestHarness // initialize partitioner hoodieTable.getHoodieView().sync(); - AbstractSparkDeltaCommitActionExecutor actionExecutor = new SparkDeleteDeltaCommitActionExecutor(context(), cfg, hoodieTable, + BaseSparkDeltaCommitActionExecutor actionExecutor = new SparkDeleteDeltaCommitActionExecutor(context(), cfg, hoodieTable, newDeleteTime, deleteRDD); actionExecutor.getUpsertPartitioner(new WorkloadProfile(buildProfile(deleteRDD))); final List> deleteStatus = jsc().parallelize(Arrays.asList(1)).map(x -> { diff --git a/hudi-spark-datasource/hudi-spark-common/src/main/java/org/apache/hudi/async/SparkStreamingAsyncClusteringService.java b/hudi-spark-datasource/hudi-spark-common/src/main/java/org/apache/hudi/async/SparkStreamingAsyncClusteringService.java index 81d880ee9..f87e16a65 100644 --- a/hudi-spark-datasource/hudi-spark-common/src/main/java/org/apache/hudi/async/SparkStreamingAsyncClusteringService.java +++ b/hudi-spark-datasource/hudi-spark-common/src/main/java/org/apache/hudi/async/SparkStreamingAsyncClusteringService.java @@ -19,8 +19,8 @@ package org.apache.hudi.async; -import org.apache.hudi.client.AbstractClusteringClient; -import org.apache.hudi.client.AbstractHoodieWriteClient; +import org.apache.hudi.client.BaseClusterer; +import org.apache.hudi.client.BaseHoodieWriteClient; import org.apache.hudi.client.HoodieSparkClusteringClient; /** @@ -31,12 +31,12 @@ public class SparkStreamingAsyncClusteringService extends AsyncClusteringService private static final long serialVersionUID = 1L; - public SparkStreamingAsyncClusteringService(AbstractHoodieWriteClient writeClient) { + public SparkStreamingAsyncClusteringService(BaseHoodieWriteClient writeClient) { super(writeClient, true); } @Override - protected AbstractClusteringClient createClusteringClient(AbstractHoodieWriteClient client) { + protected BaseClusterer createClusteringClient(BaseHoodieWriteClient client) { return new HoodieSparkClusteringClient(client); } } diff --git a/hudi-spark-datasource/hudi-spark-common/src/main/java/org/apache/hudi/async/SparkStreamingAsyncCompactService.java b/hudi-spark-datasource/hudi-spark-common/src/main/java/org/apache/hudi/async/SparkStreamingAsyncCompactService.java index 130ea7c27..2ff7b46c0 100644 --- a/hudi-spark-datasource/hudi-spark-common/src/main/java/org/apache/hudi/async/SparkStreamingAsyncCompactService.java +++ b/hudi-spark-datasource/hudi-spark-common/src/main/java/org/apache/hudi/async/SparkStreamingAsyncCompactService.java @@ -18,8 +18,8 @@ package org.apache.hudi.async; -import org.apache.hudi.client.AbstractCompactor; -import org.apache.hudi.client.AbstractHoodieWriteClient; +import org.apache.hudi.client.BaseCompactor; +import org.apache.hudi.client.BaseHoodieWriteClient; import org.apache.hudi.client.HoodieSparkCompactor; import org.apache.hudi.common.engine.HoodieEngineContext; @@ -31,12 +31,12 @@ public class SparkStreamingAsyncCompactService extends AsyncCompactService { private static final long serialVersionUID = 1L; - public SparkStreamingAsyncCompactService(HoodieEngineContext context, AbstractHoodieWriteClient client) { + public SparkStreamingAsyncCompactService(HoodieEngineContext context, BaseHoodieWriteClient client) { super(context, client, true); } @Override - protected AbstractCompactor createCompactor(AbstractHoodieWriteClient client) { + protected BaseCompactor createCompactor(BaseHoodieWriteClient client) { return new HoodieSparkCompactor(client, this.context); } }