[HUDI-1089] Refactor hudi-client to support multi-engine (#1827)

- This change breaks `hudi-client` into `hudi-client-common` and `hudi-spark-client` modules 
- Simple usages of Spark using jsc.parallelize() has been redone using EngineContext#map, EngineContext#flatMap etc
- Code changes in the PR, break classes into `BaseXYZ` parent classes with no spark dependencies living in `hudi-client-common`
- Classes on `hudi-spark-client` are named `SparkXYZ` extending the parent classes with all the Spark dependencies
- To simplify/cleanup, HoodieIndex#fetchRecordLocation has been removed and its usages in tests replaced with alternatives

Co-authored-by: Vinoth Chandar <vinoth@apache.org>

hudi-client/hudi-client-common/src/test/java/org/apache/hudi/ClientFunctionalTestSuite.java

@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hudi;
+
+import org.junit.platform.runner.JUnitPlatform;
+import org.junit.platform.suite.api.IncludeTags;
+import org.junit.platform.suite.api.SelectPackages;
+import org.junit.runner.RunWith;
+
+@RunWith(JUnitPlatform.class)
+@SelectPackages("org.apache.hudi.index")
+@IncludeTags("functional")
+public class ClientFunctionalTestSuite {
+
+}

hudi-client/hudi-client-common/src/test/java/org/apache/hudi/callback/http/TestCallbackHttpClient.java

@@ -0,0 +1,119 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.callback.http;
+
+import org.apache.http.StatusLine;
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.hudi.callback.client.http.HoodieWriteCommitHttpCallbackClient;
+import org.apache.log4j.AppenderSkeleton;
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
+import org.apache.log4j.spi.LoggingEvent;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.mockito.ArgumentCaptor;
+import org.mockito.Captor;
+import org.mockito.Mock;
+import org.mockito.junit.jupiter.MockitoExtension;
+
+import java.io.IOException;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.junit.jupiter.api.Assertions.fail;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+/**
+ * Unit test for {@link HoodieWriteCommitHttpCallbackClient}.
+ */
+@ExtendWith(MockitoExtension.class)
+public class TestCallbackHttpClient {
+
+  @Mock
+  AppenderSkeleton appender;
+
+  @Captor
+  ArgumentCaptor<LoggingEvent> logCaptor;
+
+  @Mock
+  CloseableHttpClient httpClient;
+
+  @Mock
+  CloseableHttpResponse httpResponse;
+
+  @Mock
+  StatusLine statusLine;
+
+  private void mockResponse(int statusCode) {
+    when(statusLine.getStatusCode()).thenReturn(statusCode);
+    when(httpResponse.getStatusLine()).thenReturn(statusLine);
+    try {
+      when(httpClient.execute(any())).thenReturn(httpResponse);
+    } catch (IOException e) {
+      fail(e.getMessage(), e);
+    }
+  }
+
+  @Test
+  public void sendPayloadShouldLogWhenRequestFailed() throws IOException {
+    Logger.getRootLogger().addAppender(appender);
+    when(httpClient.execute(any())).thenThrow(IOException.class);
+
+    HoodieWriteCommitHttpCallbackClient hoodieWriteCommitCallBackHttpClient =
+        new HoodieWriteCommitHttpCallbackClient("fake_api_key", "fake_url", httpClient);
+    hoodieWriteCommitCallBackHttpClient.send("{}");
+
+    verify(appender).doAppend(logCaptor.capture());
+    assertEquals("Failed to send callback.", logCaptor.getValue().getRenderedMessage());
+    assertEquals(Level.WARN, logCaptor.getValue().getLevel());
+  }
+
+  @Test
+  public void sendPayloadShouldLogUnsuccessfulSending() {
+    Logger.getRootLogger().addAppender(appender);
+    mockResponse(401);
+    when(httpResponse.toString()).thenReturn("unauthorized");
+
+    HoodieWriteCommitHttpCallbackClient hoodieWriteCommitCallBackHttpClient =
+        new HoodieWriteCommitHttpCallbackClient("fake_api_key", "fake_url", httpClient);
+    hoodieWriteCommitCallBackHttpClient.send("{}");
+
+    verify(appender).doAppend(logCaptor.capture());
+    assertEquals("Failed to send callback message. Response was unauthorized", logCaptor.getValue().getRenderedMessage());
+    assertEquals(Level.WARN, logCaptor.getValue().getLevel());
+  }
+
+  @Test
+  public void sendPayloadShouldLogSuccessfulSending() {
+    Logger.getRootLogger().addAppender(appender);
+    mockResponse(202);
+
+    HoodieWriteCommitHttpCallbackClient hoodieWriteCommitCallBackHttpClient =
+        new HoodieWriteCommitHttpCallbackClient("fake_api_key", "fake_url", httpClient);
+    hoodieWriteCommitCallBackHttpClient.send("{}");
+
+    verify(appender).doAppend(logCaptor.capture());
+    assertTrue(logCaptor.getValue().getRenderedMessage().startsWith("Sent Callback data"));
+    assertEquals(Level.INFO, logCaptor.getValue().getLevel());
+  }
+
+}

hudi-client/hudi-client-common/src/test/java/org/apache/hudi/config/TestHoodieWriteConfig.java

@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.config;
+
+import org.apache.hudi.config.HoodieWriteConfig.Builder;
+
+import org.junit.jupiter.api.Test;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Properties;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+public class TestHoodieWriteConfig {
+
+  @Test
+  public void testPropertyLoading() throws IOException {
+    Builder builder = HoodieWriteConfig.newBuilder().withPath("/tmp");
+    Map<String, String> params = new HashMap<>(3);
+    params.put(HoodieCompactionConfig.CLEANER_COMMITS_RETAINED_PROP, "1");
+    params.put(HoodieCompactionConfig.MAX_COMMITS_TO_KEEP_PROP, "5");
+    params.put(HoodieCompactionConfig.MIN_COMMITS_TO_KEEP_PROP, "2");
+    ByteArrayOutputStream outStream = saveParamsIntoOutputStream(params);
+    ByteArrayInputStream inputStream = new ByteArrayInputStream(outStream.toByteArray());
+    try {
+      builder = builder.fromInputStream(inputStream);
+    } finally {
+      outStream.close();
+      inputStream.close();
+    }
+    HoodieWriteConfig config = builder.build();
+    assertEquals(5, config.getMaxCommitsToKeep());
+    assertEquals(2, config.getMinCommitsToKeep());
+  }
+
+  private ByteArrayOutputStream saveParamsIntoOutputStream(Map<String, String> params) throws IOException {
+    Properties properties = new Properties();
+    properties.putAll(params);
+    ByteArrayOutputStream outStream = new ByteArrayOutputStream();
+    properties.store(outStream, "Saved on " + new Date(System.currentTimeMillis()));
+    return outStream;
+  }
+}

hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metrics/TestHoodieConsoleMetrics.java

@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.metrics;
+
+import org.apache.hudi.config.HoodieWriteConfig;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import static org.apache.hudi.metrics.Metrics.registerGauge;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class TestHoodieConsoleMetrics {
+
+  HoodieWriteConfig config = mock(HoodieWriteConfig.class);
+
+  @BeforeEach
+  public void start() {
+    when(config.isMetricsOn()).thenReturn(true);
+    when(config.getMetricsReporterType()).thenReturn(MetricsReporterType.CONSOLE);
+    new HoodieMetrics(config, "raw_table");
+  }
+
+  @Test
+  public void testRegisterGauge() {
+    registerGauge("metric1", 123L);
+    assertEquals("123", Metrics.getInstance().getRegistry().getGauges().get("metric1").getValue().toString());
+  }
+}

hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metrics/TestHoodieJmxMetrics.java

@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.metrics;
+
+import org.apache.hudi.common.testutils.NetworkTestUtils;
+import org.apache.hudi.config.HoodieWriteConfig;
+
+import org.junit.jupiter.api.Test;
+
+import static org.apache.hudi.metrics.Metrics.registerGauge;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+/**
+ * Test for the Jmx metrics report.
+ */
+public class TestHoodieJmxMetrics {
+
+  HoodieWriteConfig config = mock(HoodieWriteConfig.class);
+
+  @Test
+  public void testRegisterGauge() {
+    when(config.isMetricsOn()).thenReturn(true);
+    when(config.getMetricsReporterType()).thenReturn(MetricsReporterType.JMX);
+    when(config.getJmxHost()).thenReturn("localhost");
+    when(config.getJmxPort()).thenReturn(String.valueOf(NetworkTestUtils.nextFreePort()));
+    new HoodieMetrics(config, "raw_table");
+    registerGauge("jmx_metric1", 123L);
+    assertEquals("123", Metrics.getInstance().getRegistry().getGauges()
+        .get("jmx_metric1").getValue().toString());
+  }
+
+  @Test
+  public void testRegisterGaugeByRangerPort() {
+    when(config.isMetricsOn()).thenReturn(true);
+    when(config.getMetricsReporterType()).thenReturn(MetricsReporterType.JMX);
+    when(config.getJmxHost()).thenReturn("localhost");
+    when(config.getJmxPort()).thenReturn(String.valueOf(NetworkTestUtils.nextFreePort()));
+    new HoodieMetrics(config, "raw_table");
+    registerGauge("jmx_metric2", 123L);
+    assertEquals("123", Metrics.getInstance().getRegistry().getGauges()
+        .get("jmx_metric2").getValue().toString());
+  }
+}

hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metrics/TestHoodieMetrics.java

@@ -0,0 +1,161 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.metrics;
+
+import org.apache.hudi.common.model.HoodieCommitMetadata;
+import org.apache.hudi.config.HoodieWriteConfig;
+
+import com.codahale.metrics.Timer;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import java.util.Random;
+import java.util.stream.Stream;
+
+import static org.apache.hudi.metrics.Metrics.registerGauge;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class TestHoodieMetrics {
+
+  private HoodieMetrics metrics;
+
+  @BeforeEach
+  public void start() {
+    HoodieWriteConfig config = mock(HoodieWriteConfig.class);
+    when(config.isMetricsOn()).thenReturn(true);
+    when(config.getMetricsReporterType()).thenReturn(MetricsReporterType.INMEMORY);
+    metrics = new HoodieMetrics(config, "raw_table");
+  }
+
+  @Test
+  public void testRegisterGauge() {
+    registerGauge("metric1", 123L);
+    assertEquals("123", Metrics.getInstance().getRegistry().getGauges().get("metric1").getValue().toString());
+  }
+
+  @Test
+  public void testTimerCtx() throws InterruptedException {
+    Random rand = new Random();
+
+    // Index metrics
+    Timer.Context timer = metrics.getIndexCtx();
+    Thread.sleep(5); // Ensure timer duration is > 0
+    metrics.updateIndexMetrics("some_action", metrics.getDurationInMs(timer.stop()));
+    String metricName = metrics.getMetricsName("index", "some_action.duration");
+    long msec = (Long)Metrics.getInstance().getRegistry().getGauges().get(metricName).getValue();
+    assertTrue(msec > 0);
+
+    // Rollback metrics
+    timer = metrics.getRollbackCtx();
+    Thread.sleep(5); // Ensure timer duration is > 0
+    long numFilesDeleted = 1 + rand.nextInt();
+    metrics.updateRollbackMetrics(metrics.getDurationInMs(timer.stop()), numFilesDeleted);
+    metricName = metrics.getMetricsName("rollback", "duration");
+    msec = (Long)Metrics.getInstance().getRegistry().getGauges().get(metricName).getValue();
+    assertTrue(msec > 0);
+    metricName = metrics.getMetricsName("rollback", "numFilesDeleted");
+    assertEquals((long)Metrics.getInstance().getRegistry().getGauges().get(metricName).getValue(), numFilesDeleted);
+
+    // Clean metrics
+    timer = metrics.getRollbackCtx();
+    Thread.sleep(5); // Ensure timer duration is > 0
+    numFilesDeleted = 1 + rand.nextInt();
+    metrics.updateCleanMetrics(metrics.getDurationInMs(timer.stop()), (int)numFilesDeleted);
+    metricName = metrics.getMetricsName("clean", "duration");
+    msec = (Long)Metrics.getInstance().getRegistry().getGauges().get(metricName).getValue();
+    assertTrue(msec > 0);
+    metricName = metrics.getMetricsName("clean", "numFilesDeleted");
+    assertEquals((long)Metrics.getInstance().getRegistry().getGauges().get(metricName).getValue(), numFilesDeleted);
+
+    // Finalize metrics
+    timer = metrics.getFinalizeCtx();
+    Thread.sleep(5); // Ensure timer duration is > 0
+    long numFilesFinalized = 1 + rand.nextInt();
+    metrics.updateFinalizeWriteMetrics(metrics.getDurationInMs(timer.stop()), (int)numFilesFinalized);
+    metricName = metrics.getMetricsName("finalize", "duration");
+    msec = (Long)Metrics.getInstance().getRegistry().getGauges().get(metricName).getValue();
+    assertTrue(msec > 0);
+    metricName = metrics.getMetricsName("finalize", "numFilesFinalized");
+    assertEquals((long)Metrics.getInstance().getRegistry().getGauges().get(metricName).getValue(), numFilesFinalized);
+
+    // Commit / deltacommit / compaction metrics
+    Stream.of("commit", "deltacommit", "compaction").forEach(action -> {
+      Timer.Context commitTimer = action.equals("commit") ? metrics.getCommitCtx() :
+          action.equals("deltacommit") ? metrics.getDeltaCommitCtx() : metrics.getCompactionCtx();
+
+      try {
+        // Ensure timer duration is > 0
+        Thread.sleep(5);
+      } catch (InterruptedException e) {
+        e.printStackTrace();
+      }
+      long randomValue = 1 + rand.nextInt();
+      HoodieCommitMetadata metadata = mock(HoodieCommitMetadata.class);
+      when(metadata.fetchTotalPartitionsWritten()).thenReturn(randomValue + 1);
+      when(metadata.fetchTotalFilesInsert()).thenReturn(randomValue + 2);
+      when(metadata.fetchTotalFilesUpdated()).thenReturn(randomValue + 3);
+      when(metadata.fetchTotalRecordsWritten()).thenReturn(randomValue + 4);
+      when(metadata.fetchTotalUpdateRecordsWritten()).thenReturn(randomValue + 5);
+      when(metadata.fetchTotalInsertRecordsWritten()).thenReturn(randomValue + 6);
+      when(metadata.fetchTotalBytesWritten()).thenReturn(randomValue + 7);
+      when(metadata.getTotalScanTime()).thenReturn(randomValue + 8);
+      when(metadata.getTotalCreateTime()).thenReturn(randomValue + 9);
+      when(metadata.getTotalUpsertTime()).thenReturn(randomValue + 10);
+      when(metadata.getTotalCompactedRecordsUpdated()).thenReturn(randomValue + 11);
+      when(metadata.getTotalLogFilesCompacted()).thenReturn(randomValue + 12);
+      when(metadata.getTotalLogFilesSize()).thenReturn(randomValue + 13);
+      metrics.updateCommitMetrics(randomValue + 14, commitTimer.stop(), metadata, action);
+
+      String metricname = metrics.getMetricsName(action, "duration");
+      long duration = (Long)Metrics.getInstance().getRegistry().getGauges().get(metricname).getValue();
+      assertTrue(duration > 0);
+      metricname = metrics.getMetricsName(action, "totalPartitionsWritten");
+      assertEquals((long)Metrics.getInstance().getRegistry().getGauges().get(metricname).getValue(), metadata.fetchTotalPartitionsWritten());
+      metricname = metrics.getMetricsName(action, "totalFilesInsert");
+      assertEquals((long)Metrics.getInstance().getRegistry().getGauges().get(metricname).getValue(), metadata.fetchTotalFilesInsert());
+      metricname = metrics.getMetricsName(action, "totalFilesUpdate");
+      assertEquals((long)Metrics.getInstance().getRegistry().getGauges().get(metricname).getValue(), metadata.fetchTotalFilesUpdated());
+      metricname = metrics.getMetricsName(action, "totalRecordsWritten");
+      assertEquals((long)Metrics.getInstance().getRegistry().getGauges().get(metricname).getValue(), metadata.fetchTotalRecordsWritten());
+      metricname = metrics.getMetricsName(action, "totalUpdateRecordsWritten");
+      assertEquals((long)Metrics.getInstance().getRegistry().getGauges().get(metricname).getValue(), metadata.fetchTotalUpdateRecordsWritten());
+      metricname = metrics.getMetricsName(action, "totalInsertRecordsWritten");
+      assertEquals((long)Metrics.getInstance().getRegistry().getGauges().get(metricname).getValue(), metadata.fetchTotalInsertRecordsWritten());
+      metricname = metrics.getMetricsName(action, "totalBytesWritten");
+      assertEquals((long)Metrics.getInstance().getRegistry().getGauges().get(metricname).getValue(), metadata.fetchTotalBytesWritten());
+      metricname = metrics.getMetricsName(action, "commitTime");
+      assertEquals((long)Metrics.getInstance().getRegistry().getGauges().get(metricname).getValue(), randomValue + 14);
+      metricname = metrics.getMetricsName(action, "totalScanTime");
+      assertEquals(Metrics.getInstance().getRegistry().getGauges().get(metricname).getValue(), metadata.getTotalScanTime());
+      metricname = metrics.getMetricsName(action, "totalCreateTime");
+      assertEquals(Metrics.getInstance().getRegistry().getGauges().get(metricname).getValue(), metadata.getTotalCreateTime());
+      metricname = metrics.getMetricsName(action, "totalUpsertTime");
+      assertEquals(Metrics.getInstance().getRegistry().getGauges().get(metricname).getValue(), metadata.getTotalUpsertTime());
+      metricname = metrics.getMetricsName(action, "totalCompactedRecordsUpdated");
+      assertEquals(Metrics.getInstance().getRegistry().getGauges().get(metricname).getValue(), metadata.getTotalCompactedRecordsUpdated());
+      metricname = metrics.getMetricsName(action, "totalLogFilesCompacted");
+      assertEquals(Metrics.getInstance().getRegistry().getGauges().get(metricname).getValue(), metadata.getTotalLogFilesCompacted());
+      metricname = metrics.getMetricsName(action, "totalLogFilesSize");
+      assertEquals(Metrics.getInstance().getRegistry().getGauges().get(metricname).getValue(), metadata.getTotalLogFilesSize());
+    });
+  }
+}

hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metrics/TestMetricsReporterFactory.java

@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hudi.metrics;
+
+import org.apache.hudi.config.HoodieWriteConfig;
+
+import com.codahale.metrics.MetricRegistry;
+import org.apache.hudi.exception.HoodieException;
+import org.apache.hudi.metrics.userdefined.AbstractUserDefinedMetricsReporter;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.mockito.Mock;
+import org.mockito.junit.jupiter.MockitoExtension;
+
+import java.io.Closeable;
+import java.util.Properties;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.Mockito.when;
+
+@ExtendWith(MockitoExtension.class)
+public class TestMetricsReporterFactory {
+
+  @Mock
+  HoodieWriteConfig config;
+
+  @Mock
+  MetricRegistry registry;
+
+  @Test
+  public void metricsReporterFactoryShouldReturnReporter() {
+    when(config.getMetricsReporterType()).thenReturn(MetricsReporterType.INMEMORY);
+    MetricsReporter reporter = MetricsReporterFactory.createReporter(config, registry);
+    assertTrue(reporter instanceof InMemoryMetricsReporter);
+  }
+
+  @Test
+  public void metricsReporterFactoryShouldReturnUserDefinedReporter() {
+    when(config.getMetricReporterClassName()).thenReturn(DummyMetricsReporter.class.getName());
+
+    Properties props = new Properties();
+    props.setProperty("testKey", "testValue");
+
+    when(config.getProps()).thenReturn(props);
+    MetricsReporter reporter = MetricsReporterFactory.createReporter(config, registry);
+    assertTrue(reporter instanceof AbstractUserDefinedMetricsReporter);
+    assertEquals(props, ((DummyMetricsReporter) reporter).getProps());
+    assertEquals(registry, ((DummyMetricsReporter) reporter).getRegistry());
+  }
+
+  @Test
+  public void metricsReporterFactoryShouldThrowExceptionWhenMetricsReporterClassIsIllegal() {
+    when(config.getMetricReporterClassName()).thenReturn(IllegalTestMetricsReporter.class.getName());
+    when(config.getProps()).thenReturn(new Properties());
+    assertThrows(HoodieException.class, () -> MetricsReporterFactory.createReporter(config, registry));
+  }
+
+  public static class DummyMetricsReporter extends AbstractUserDefinedMetricsReporter {
+
+    public DummyMetricsReporter(Properties props, MetricRegistry registry) {
+      super(props, registry);
+    }
+
+    @Override
+    public void start() {}
+
+    @Override
+    public void report() {}
+
+    @Override
+    public Closeable getReporter() {
+      return null;
+    }
+
+    @Override
+    public void stop() {}
+  }
+
+  public static class IllegalTestMetricsReporter {
+
+    public IllegalTestMetricsReporter(Properties props, MetricRegistry registry) {}
+  }
+}
+
+

hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metrics/datadog/TestDatadogHttpClient.java

@@ -0,0 +1,152 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.metrics.datadog;
+
+import org.apache.hudi.metrics.datadog.DatadogHttpClient.ApiSite;
+
+import org.apache.http.StatusLine;
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.log4j.AppenderSkeleton;
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
+import org.apache.log4j.spi.LoggingEvent;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.Arguments;
+import org.junit.jupiter.params.provider.MethodSource;
+import org.mockito.ArgumentCaptor;
+import org.mockito.Captor;
+import org.mockito.Mock;
+import org.mockito.junit.jupiter.MockitoExtension;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.List;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.junit.jupiter.api.Assertions.fail;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+@ExtendWith(MockitoExtension.class)
+public class TestDatadogHttpClient {
+
+  @Mock
+  AppenderSkeleton appender;
+
+  @Captor
+  ArgumentCaptor<LoggingEvent> logCaptor;
+
+  @Mock
+  CloseableHttpClient httpClient;
+
+  @Mock
+  CloseableHttpResponse httpResponse;
+
+  @Mock
+  StatusLine statusLine;
+
+  private void mockResponse(int statusCode) {
+    when(statusLine.getStatusCode()).thenReturn(statusCode);
+    when(httpResponse.getStatusLine()).thenReturn(statusLine);
+    try {
+      when(httpClient.execute(any())).thenReturn(httpResponse);
+    } catch (IOException e) {
+      fail(e.getMessage(), e);
+    }
+  }
+
+  @Test
+  public void validateApiKeyShouldThrowExceptionWhenRequestFailed() throws IOException {
+    when(httpClient.execute(any())).thenThrow(IOException.class);
+
+    Throwable t = assertThrows(IllegalStateException.class, () -> {
+      new DatadogHttpClient(ApiSite.EU, "foo", false, httpClient);
+    });
+    assertEquals("Failed to connect to Datadog to validate API key.", t.getMessage());
+  }
+
+  @Test
+  public void validateApiKeyShouldThrowExceptionWhenResponseNotSuccessful() {
+    mockResponse(500);
+
+    Throwable t = assertThrows(IllegalStateException.class, () -> {
+      new DatadogHttpClient(ApiSite.EU, "foo", false, httpClient);
+    });
+    assertEquals("API key is invalid.", t.getMessage());
+  }
+
+  @Test
+  public void sendPayloadShouldLogWhenRequestFailed() throws IOException {
+    Logger.getRootLogger().addAppender(appender);
+    when(httpClient.execute(any())).thenThrow(IOException.class);
+
+    DatadogHttpClient ddClient = new DatadogHttpClient(ApiSite.US, "foo", true, httpClient);
+    ddClient.send("{}");
+
+    verify(appender).doAppend(logCaptor.capture());
+    assertEquals("Failed to send to Datadog.", logCaptor.getValue().getRenderedMessage());
+    assertEquals(Level.WARN, logCaptor.getValue().getLevel());
+  }
+
+  @Test
+  public void sendPayloadShouldLogUnsuccessfulSending() {
+    Logger.getRootLogger().addAppender(appender);
+    mockResponse(401);
+    when(httpResponse.toString()).thenReturn("unauthorized");
+
+    DatadogHttpClient ddClient = new DatadogHttpClient(ApiSite.US, "foo", true, httpClient);
+    ddClient.send("{}");
+
+    verify(appender).doAppend(logCaptor.capture());
+    assertEquals("Failed to send to Datadog. Response was unauthorized", logCaptor.getValue().getRenderedMessage());
+    assertEquals(Level.WARN, logCaptor.getValue().getLevel());
+  }
+
+  @Test
+  public void sendPayloadShouldLogSuccessfulSending() {
+    Logger.getRootLogger().addAppender(appender);
+    mockResponse(202);
+
+    DatadogHttpClient ddClient = new DatadogHttpClient(ApiSite.US, "foo", true, httpClient);
+    ddClient.send("{}");
+    
+    verify(appender).doAppend(logCaptor.capture());
+    assertTrue(logCaptor.getValue().getRenderedMessage().startsWith("Sent metrics data"));
+    assertEquals(Level.DEBUG, logCaptor.getValue().getLevel());
+  }
+
+  public static List<Arguments> getApiSiteAndDomain() {
+    return Arrays.asList(
+        Arguments.of("US", "com"),
+        Arguments.of("EU", "eu")
+    );
+  }
+
+  @ParameterizedTest
+  @MethodSource("getApiSiteAndDomain")
+  public void testApiSiteReturnCorrectDomain(String apiSite, String domain) {
+    assertEquals(domain, ApiSite.valueOf(apiSite).getDomain());
+  }
+}

hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metrics/datadog/TestDatadogMetricsReporter.java

@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.metrics.datadog;
+
+import org.apache.hudi.config.HoodieWriteConfig;
+import org.apache.hudi.metrics.datadog.DatadogHttpClient.ApiSite;
+
+import com.codahale.metrics.MetricRegistry;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.mockito.Mock;
+import org.mockito.junit.jupiter.MockitoExtension;
+
+import java.util.Arrays;
+
+import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.mockito.Mockito.when;
+
+@ExtendWith(MockitoExtension.class)
+public class TestDatadogMetricsReporter {
+
+  @Mock
+  HoodieWriteConfig config;
+
+  @Mock
+  MetricRegistry registry;
+
+  @Test
+  public void instantiationShouldFailWhenNoApiKey() {
+    when(config.getDatadogApiKey()).thenReturn("");
+    Throwable t = assertThrows(IllegalStateException.class, () -> {
+      new DatadogMetricsReporter(config, registry);
+    });
+    assertEquals("Datadog cannot be initialized: API key is null or empty.", t.getMessage());
+  }
+
+  @Test
+  public void instantiationShouldFailWhenNoMetricPrefix() {
+    when(config.getDatadogApiKey()).thenReturn("foo");
+    when(config.getDatadogMetricPrefix()).thenReturn("");
+    Throwable t = assertThrows(IllegalStateException.class, () -> {
+      new DatadogMetricsReporter(config, registry);
+    });
+    assertEquals("Datadog cannot be initialized: Metric prefix is null or empty.", t.getMessage());
+  }
+
+  @Test
+  public void instantiationShouldSucceed() {
+    when(config.getDatadogApiSite()).thenReturn(ApiSite.EU);
+    when(config.getDatadogApiKey()).thenReturn("foo");
+    when(config.getDatadogApiKeySkipValidation()).thenReturn(true);
+    when(config.getDatadogMetricPrefix()).thenReturn("bar");
+    when(config.getDatadogMetricHost()).thenReturn("foo");
+    when(config.getDatadogMetricTags()).thenReturn(Arrays.asList("baz", "foo"));
+    assertDoesNotThrow(() -> {
+      new DatadogMetricsReporter(config, registry);
+    });
+  }
+}

hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metrics/datadog/TestDatadogReporter.java

@@ -0,0 +1,105 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.metrics.datadog;
+
+import org.apache.hudi.common.util.Option;
+import org.apache.hudi.metrics.datadog.DatadogReporter.MetricType;
+import org.apache.hudi.metrics.datadog.DatadogReporter.PayloadBuilder;
+
+import com.codahale.metrics.MetricFilter;
+import com.codahale.metrics.MetricRegistry;
+import org.apache.log4j.AppenderSkeleton;
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
+import org.apache.log4j.spi.LoggingEvent;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.mockito.ArgumentCaptor;
+import org.mockito.Captor;
+import org.mockito.Mock;
+import org.mockito.junit.jupiter.MockitoExtension;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.concurrent.TimeUnit;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.mockito.Mockito.doThrow;
+import static org.mockito.Mockito.verify;
+
+@ExtendWith(MockitoExtension.class)
+public class TestDatadogReporter {
+
+  @Mock
+  AppenderSkeleton appender;
+
+  @Captor
+  ArgumentCaptor<LoggingEvent> logCaptor;
+
+  @Mock
+  MetricRegistry registry;
+
+  @Mock
+  DatadogHttpClient client;
+
+  @Test
+  public void stopShouldCloseEnclosedClient() throws IOException {
+    new DatadogReporter(registry, client, "foo", Option.empty(), Option.empty(),
+        MetricFilter.ALL, TimeUnit.SECONDS, TimeUnit.SECONDS).stop();
+
+    verify(client).close();
+  }
+
+  @Test
+  public void stopShouldLogWhenEnclosedClientFailToClose() throws IOException {
+    Logger.getRootLogger().addAppender(appender);
+    doThrow(IOException.class).when(client).close();
+
+    new DatadogReporter(registry, client, "foo", Option.empty(), Option.empty(),
+        MetricFilter.ALL, TimeUnit.SECONDS, TimeUnit.SECONDS).stop();
+
+    verify(appender).doAppend(logCaptor.capture());
+    assertEquals("Error disconnecting from Datadog.", logCaptor.getValue().getRenderedMessage());
+    assertEquals(Level.WARN, logCaptor.getValue().getLevel());
+  }
+
+  @Test
+  public void prefixShouldPrepend() {
+    DatadogReporter reporter = new DatadogReporter(
+        registry, client, "foo", Option.empty(), Option.empty(),
+        MetricFilter.ALL, TimeUnit.SECONDS, TimeUnit.SECONDS);
+    assertEquals("foo.bar", reporter.prefix("bar"));
+  }
+
+  @Test
+  public void payloadBuilderShouldBuildExpectedPayloadString() {
+    String payload = new PayloadBuilder()
+        .withMetricType(MetricType.gauge)
+        .addGauge("foo", 0, 0)
+        .addGauge("bar", 1, 999)
+        .withHost("xhost")
+        .withTags(Arrays.asList("tag1", "tag2"))
+        .build();
+    assertEquals(
+        "{\"series\":["
+            + "{\"metric\":\"foo\",\"points\":[[0,0]],\"host\":\"xhost\",\"tags\":[\"tag1\",\"tag2\"]},"
+            + "{\"metric\":\"bar\",\"points\":[[1,999]],\"host\":\"xhost\",\"tags\":[\"tag1\",\"tag2\"]}]}",
+        payload);
+  }
+}

hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metrics/prometheus/TestPrometheusReporter.java

@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.metrics.prometheus;
+
+import org.apache.hudi.config.HoodieWriteConfig;
+import org.apache.hudi.metrics.HoodieMetrics;
+import org.apache.hudi.metrics.MetricsReporterType;
+import org.junit.jupiter.api.Test;
+
+import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class TestPrometheusReporter {
+
+  HoodieWriteConfig config = mock(HoodieWriteConfig.class);
+
+  @Test
+  public void testRegisterGauge() {
+    when(config.isMetricsOn()).thenReturn(true);
+    when(config.getMetricsReporterType()).thenReturn(MetricsReporterType.PROMETHEUS);
+    when(config.getPrometheusPort()).thenReturn(9090);
+    assertDoesNotThrow(() -> {
+      new HoodieMetrics(config, "raw_table");
+    });
+  }
+}

hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metrics/prometheus/TestPushGateWayReporter.java

@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.metrics.prometheus;
+
+import org.apache.hudi.config.HoodieWriteConfig;
+import org.apache.hudi.metrics.HoodieMetrics;
+import org.apache.hudi.metrics.Metrics;
+import org.apache.hudi.metrics.MetricsReporterType;
+import org.junit.jupiter.api.Test;
+
+import static org.apache.hudi.metrics.Metrics.registerGauge;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class TestPushGateWayReporter {
+
+  HoodieWriteConfig config = mock(HoodieWriteConfig.class);
+
+  @Test
+  public void testRegisterGauge() {
+    when(config.isMetricsOn()).thenReturn(true);
+    when(config.getMetricsReporterType()).thenReturn(MetricsReporterType.PROMETHEUS_PUSHGATEWAY);
+    when(config.getPushGatewayHost()).thenReturn("localhost");
+    when(config.getPushGatewayPort()).thenReturn(9091);
+    new HoodieMetrics(config, "raw_table");
+    registerGauge("pushGateWayReporter_metric", 123L);
+    assertEquals("123", Metrics.getInstance().getRegistry().getGauges()
+        .get("pushGateWayReporter_metric").getValue().toString());
+  }
+}

hudi-client/hudi-client-common/src/test/java/org/apache/hudi/testutils/providers/DFSProvider.java

@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hudi.testutils.providers;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.DistributedFileSystem;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
+
+public interface DFSProvider {
+
+  MiniDFSCluster dfsCluster();
+
+  DistributedFileSystem dfs();
+
+  Path dfsBasePath();
+
+}

hudi-client/hudi-client-common/src/test/java/org/apache/hudi/testutils/providers/HoodieEngineContextProvider.java

@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hudi.testutils.providers;
+
+import org.apache.hudi.client.common.HoodieEngineContext;
+
+public interface HoodieEngineContextProvider {
+  HoodieEngineContext context();
+}

hudi-client/hudi-client-common/src/test/java/org/apache/hudi/testutils/providers/HoodieMetaClientProvider.java

@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hudi.testutils.providers;
+
+import org.apache.hudi.common.table.HoodieTableMetaClient;
+import org.apache.hudi.common.table.timeline.HoodieTimeline;
+import org.apache.hudi.common.table.view.HoodieTableFileSystemView;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+
+import java.io.IOException;
+import java.util.Properties;
+
+public interface HoodieMetaClientProvider {
+
+  HoodieTableMetaClient getHoodieMetaClient(Configuration hadoopConf, String basePath, Properties props) throws IOException;
+
+  default HoodieTableFileSystemView getHoodieTableFileSystemView(
+      HoodieTableMetaClient metaClient, HoodieTimeline visibleActiveTimeline, FileStatus[] fileStatuses) {
+    return new HoodieTableFileSystemView(metaClient, visibleActiveTimeline, fileStatuses);
+  }
+}

hudi-client/hudi-client-common/src/test/java/org/apache/hudi/testutils/providers/HoodieWriteClientProvider.java

@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hudi.testutils.providers;
+
+import org.apache.hudi.client.AbstractHoodieWriteClient;
+import org.apache.hudi.config.HoodieWriteConfig;
+
+import java.io.IOException;
+
+public interface HoodieWriteClientProvider {
+
+  AbstractHoodieWriteClient getHoodieWriteClient(HoodieWriteConfig cfg) throws IOException;
+}

hudi-client/hudi-client-common/src/test/resources/exampleEvolvedSchema.txt

@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+{
+    "namespace": "example.schema",
+    "type": "record",
+    "name": "trip",
+    "fields": [
+        {
+            "name": "number",
+            "type": ["int", "null"]
+        },
+        {
+            "name": "time",
+            "type": "string"
+        },
+        {
+            "name": "_row_key",
+            "type": "string"
+        },
+        {
+            "name": "added_field",
+            "type": ["int", "null"]
+        }
+    ]
+}

hudi-client/hudi-client-common/src/test/resources/exampleSchema.txt

@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+{
+    "namespace": "example.schema",
+    "type": "record",
+    "name": "trip",
+    "fields": [
+        {
+            "name": "_row_key",
+            "type": "string"
+        },
+        {
+            "name": "time",
+            "type": "string"
+        },
+        {
+            "name": "number",
+            "type": ["int", "null"]
+        }
+    ]
+}

hudi-client/hudi-client-common/src/test/resources/log4j-surefire-quiet.properties

@@ -0,0 +1,30 @@
+###
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+###
+log4j.rootLogger=WARN, CONSOLE
+log4j.logger.org.apache.hudi=DEBUG
+log4j.logger.org.apache.hadoop.hbase=ERROR
+
+# CONSOLE is set to be a ConsoleAppender.
+log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender
+# CONSOLE uses PatternLayout.
+log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout
+log4j.appender.CONSOLE.layout.ConversionPattern=[%-5p] %d %c %x - %m%n
+log4j.appender.CONSOLE.filter.a=org.apache.log4j.varia.LevelRangeFilter
+log4j.appender.CONSOLE.filter.a.AcceptOnMatch=true
+log4j.appender.CONSOLE.filter.a.LevelMin=WARN
+log4j.appender.CONSOLE.filter.a.LevelMax=FATAL

hudi-client/hudi-client-common/src/test/resources/log4j-surefire.properties

@@ -0,0 +1,31 @@
+###
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+###
+log4j.rootLogger=WARN, CONSOLE
+log4j.logger.org.apache=INFO
+log4j.logger.org.apache.hudi=DEBUG
+log4j.logger.org.apache.hadoop.hbase=ERROR
+
+# A1 is set to be a ConsoleAppender.
+log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender
+# A1 uses PatternLayout.
+log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout
+log4j.appender.CONSOLE.layout.ConversionPattern=%-4r [%t] %-5p %c %x - %m%n
+log4j.appender.CONSOLE.filter.a=org.apache.log4j.varia.LevelRangeFilter
+log4j.appender.CONSOLE.filter.a.AcceptOnMatch=true
+log4j.appender.CONSOLE.filter.a.LevelMin=WARN
+log4j.appender.CONSOLE.filter.a.LevelMax=FATAL