[HUDI-2883] Refactor hive sync tool / config to use reflection and standardize configs (#4175)

- Refactor hive sync tool / config to use reflection and standardize configs Co-authored-by: sivabalan <n.siva.b@gmail.com> Co-authored-by: Rajesh Mahindra <rmahindra@Rajeshs-MacBook-Pro.local> Co-authored-by: Raymond Xu <2701446+xushiyan@users.noreply.github.com>
2022-03-21 19:56:31 -07:00
parent 9b6e138af2
commit 5f570ea151
43 changed files with 1521 additions and 1217 deletions
--- a/hudi-sync/hudi-sync-common/src/test/java/org/apache/hudi/sync/common/util/TestSyncUtilHelpers.java
+++ b/hudi-sync/hudi-sync-common/src/test/java/org/apache/hudi/sync/common/util/TestSyncUtilHelpers.java
@@ -0,0 +1,124 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.sync.common.util;
+
+import org.apache.hudi.common.config.TypedProperties;
+import org.apache.hudi.common.fs.FSUtils;
+import org.apache.hudi.exception.HoodieException;
+import org.apache.hudi.sync.common.AbstractSyncTool;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import java.io.IOException;
+import java.util.Properties;
+
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+public class TestSyncUtilHelpers {
+  private static final String BASE_PATH = "/tmp/test";
+  private static final String BASE_FORMAT = "PARQUET";
+
+  private Configuration hadoopConf;
+  private FileSystem fileSystem;
+
+  @BeforeEach
+  public void setUp() throws IOException {
+    fileSystem = FSUtils.getFs(BASE_PATH, new Configuration());
+    hadoopConf = fileSystem.getConf();
+  }
+
+  @Test
+  public void testCreateValidSyncClass() {
+    AbstractSyncTool metaSyncTool = SyncUtilHelpers.instantiateMetaSyncTool(
+        ValidMetaSyncClass.class.getName(),
+        new TypedProperties(),
+        hadoopConf,
+        fileSystem,
+        BASE_PATH,
+        BASE_FORMAT
+    );
+    assertTrue(metaSyncTool instanceof ValidMetaSyncClass);
+  }
+
+  /**
+   * Ensure it still works for the deprecated constructor of {@link AbstractSyncTool}
+   * as we implemented the fallback.
+   */
+  @Test
+  public void testCreateDeprecatedSyncClass() {
+    Properties properties = new Properties();
+    AbstractSyncTool deprecatedMetaSyncClass = SyncUtilHelpers.instantiateMetaSyncTool(
+        DeprecatedMetaSyncClass.class.getName(),
+        new TypedProperties(properties),
+        hadoopConf,
+        fileSystem,
+        BASE_PATH,
+        BASE_FORMAT
+    );
+    assertTrue(deprecatedMetaSyncClass instanceof DeprecatedMetaSyncClass);
+  }
+
+  @Test
+  public void testCreateInvalidSyncClass() {
+    Exception exception = assertThrows(HoodieException.class, () -> {
+      SyncUtilHelpers.instantiateMetaSyncTool(
+          InvalidSyncClass.class.getName(),
+          new TypedProperties(),
+          hadoopConf,
+          fileSystem,
+          BASE_PATH,
+          BASE_FORMAT
+      );
+    });
+
+    String expectedMessage = "Could not load meta sync class " + InvalidSyncClass.class.getName();
+    assertTrue(exception.getMessage().contains(expectedMessage));
+
+  }
+
+  public static class ValidMetaSyncClass extends AbstractSyncTool {
+    public ValidMetaSyncClass(TypedProperties props, Configuration conf, FileSystem fs) {
+      super(props, conf, fs);
+    }
+
+    @Override
+    public void syncHoodieTable() {
+      throw new HoodieException("Method unimplemented as its a test class");
+    }
+  }
+
+  public static class DeprecatedMetaSyncClass extends AbstractSyncTool {
+    public DeprecatedMetaSyncClass(Properties props, FileSystem fileSystem) {
+      super(props, fileSystem);
+    }
+
+    @Override
+    public void syncHoodieTable() {
+      throw new HoodieException("Method unimplemented as its a test class");
+    }
+  }
+
+  public static class InvalidSyncClass {
+    public InvalidSyncClass(Properties props) {
+    }
+  }
+}
--- a/hudi-sync/hudi-sync-common/src/test/resources/log4j-surefire-quiet.properties
+++ b/hudi-sync/hudi-sync-common/src/test/resources/log4j-surefire-quiet.properties
@@ -0,0 +1,29 @@
+###
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+###
+log4j.rootLogger=WARN, CONSOLE
+log4j.logger.org.apache.hudi=DEBUG
+
+# CONSOLE is set to be a ConsoleAppender.
+log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender
+# CONSOLE uses PatternLayout.
+log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout
+log4j.appender.CONSOLE.layout.ConversionPattern=[%-5p] %d %c %x - %m%n
+log4j.appender.CONSOLE.filter.a=org.apache.log4j.varia.LevelRangeFilter
+log4j.appender.CONSOLE.filter.a.AcceptOnMatch=true
+log4j.appender.CONSOLE.filter.a.LevelMin=WARN
+log4j.appender.CONSOLE.filter.a.LevelMax=FATAL
--- a/hudi-sync/hudi-sync-common/src/test/resources/log4j-surefire.properties
+++ b/hudi-sync/hudi-sync-common/src/test/resources/log4j-surefire.properties
@@ -0,0 +1,30 @@
+###
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+###
+log4j.rootLogger=WARN, CONSOLE
+log4j.logger.org.apache=INFO
+log4j.logger.org.apache.hudi=DEBUG
+
+# A1 is set to be a ConsoleAppender.
+log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender
+# A1 uses PatternLayout.
+log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout
+log4j.appender.CONSOLE.layout.ConversionPattern=%-4r [%t] %-5p %c %x - %m%n
+log4j.appender.CONSOLE.filter.a=org.apache.log4j.varia.LevelRangeFilter
+log4j.appender.CONSOLE.filter.a.AcceptOnMatch=true
+log4j.appender.CONSOLE.filter.a.LevelMin=WARN
+log4j.appender.CONSOLE.filter.a.LevelMax=FATAL