lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-4107] Added --sync-tool-classes config option in HoodieMultiTableDeltaStreamer (#5597)

Browse Source 
* added --sync-tool-classes config option in multitable delta streamer

* added a testcase to assert if syncClientToolClassNames is getting picked to the deltastreamer execution context
This commit is contained in:
Kumud Kumar Srivatsava Tirupati
2022-05-31 20:27:50 +05:30
committed by GitHub
parent 918c4f4e0b
commit 795a99ba73
4 changed files with 22 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
hudi-utilities/src/test/java/org/apache/hudi/utilities/functional/TestHoodieMultiTableDeltaStreamer.java
View File

@@ -72,10 +72,19 @@ public class TestHoodieMultiTableDeltaStreamer extends HoodieDeltaStreamerTestBa
}
config.enableHiveSync = enableHiveSync;
config.enableMetaSync = enableMetaSync;
config.syncClientToolClassNames = "com.example.DummySyncTool1,com.example.DummySyncTool2";
return config;
}
}
@Test
public void testMetaSyncConfig() throws IOException {
HoodieMultiTableDeltaStreamer.Config cfg = TestHelpers.getConfig(PROPS_FILENAME_TEST_SOURCE1, dfsBasePath + "/config", TestDataSource.class.getName(), true, true, null);
HoodieMultiTableDeltaStreamer streamer = new HoodieMultiTableDeltaStreamer(cfg, jsc);
TableExecutionContext executionContext = streamer.getTableExecutionContexts().get(1);
assertEquals("com.example.DummySyncTool1,com.example.DummySyncTool2", executionContext.getConfig().syncClientToolClassNames);
}
@Test
public void testInvalidHiveSyncProps() throws IOException {
HoodieMultiTableDeltaStreamer.Config cfg = TestHelpers.getConfig(PROPS_INVALID_HIVE_SYNC_TEST_SOURCE1, dfsBasePath + "/config", TestDataSource.class.getName(), true, true, null);