Optimize hoodie client after implementat auto closeable interface
This commit is contained in:
@@ -65,9 +65,6 @@ public class TestHoodieClientBase extends HoodieClientTestHarness {
|
|||||||
|
|
||||||
protected static Logger logger = LogManager.getLogger(TestHoodieClientBase.class);
|
protected static Logger logger = LogManager.getLogger(TestHoodieClientBase.class);
|
||||||
|
|
||||||
private HoodieWriteClient writeClient;
|
|
||||||
private HoodieReadClient readClient;
|
|
||||||
|
|
||||||
protected HoodieWriteClient getHoodieWriteClient(HoodieWriteConfig cfg) {
|
protected HoodieWriteClient getHoodieWriteClient(HoodieWriteConfig cfg) {
|
||||||
return getHoodieWriteClient(cfg, false);
|
return getHoodieWriteClient(cfg, false);
|
||||||
}
|
}
|
||||||
@@ -78,29 +75,11 @@ public class TestHoodieClientBase extends HoodieClientTestHarness {
|
|||||||
|
|
||||||
protected HoodieWriteClient getHoodieWriteClient(HoodieWriteConfig cfg, boolean rollbackInflightCommit,
|
protected HoodieWriteClient getHoodieWriteClient(HoodieWriteConfig cfg, boolean rollbackInflightCommit,
|
||||||
HoodieIndex index) {
|
HoodieIndex index) {
|
||||||
closeWriteClient();
|
return new HoodieWriteClient(jsc, cfg, rollbackInflightCommit, index);
|
||||||
writeClient = new HoodieWriteClient(jsc, cfg, rollbackInflightCommit, index);
|
|
||||||
return writeClient;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
protected HoodieReadClient getHoodieReadClient(String basePath) {
|
protected HoodieReadClient getHoodieReadClient(String basePath) {
|
||||||
closeReadClient();
|
return new HoodieReadClient(jsc, basePath);
|
||||||
readClient = new HoodieReadClient(jsc, basePath);
|
|
||||||
return readClient;
|
|
||||||
}
|
|
||||||
|
|
||||||
private void closeWriteClient() {
|
|
||||||
if (null != writeClient) {
|
|
||||||
writeClient.close();
|
|
||||||
writeClient = null;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private void closeReadClient() {
|
|
||||||
if (null != readClient) {
|
|
||||||
readClient.close();
|
|
||||||
readClient = null;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|||||||
@@ -49,7 +49,6 @@ public class TestMultiFS extends HoodieClientTestHarness {
|
|||||||
private static Logger logger = LogManager.getLogger(TestMultiFS.class);
|
private static Logger logger = LogManager.getLogger(TestMultiFS.class);
|
||||||
private String tablePath = "file:///tmp/hoodie/sample-table";
|
private String tablePath = "file:///tmp/hoodie/sample-table";
|
||||||
protected String tableName = "hoodie_rt";
|
protected String tableName = "hoodie_rt";
|
||||||
private HoodieWriteClient hdfsWriteClient;
|
|
||||||
private String tableType = HoodieTableType.COPY_ON_WRITE.name();
|
private String tableType = HoodieTableType.COPY_ON_WRITE.name();
|
||||||
|
|
||||||
@Before
|
@Before
|
||||||
@@ -67,11 +66,7 @@ public class TestMultiFS extends HoodieClientTestHarness {
|
|||||||
}
|
}
|
||||||
|
|
||||||
private HoodieWriteClient getHoodieWriteClient(HoodieWriteConfig config) throws Exception {
|
private HoodieWriteClient getHoodieWriteClient(HoodieWriteConfig config) throws Exception {
|
||||||
if (null != hdfsWriteClient) {
|
return new HoodieWriteClient(jsc, config);
|
||||||
hdfsWriteClient.close();
|
|
||||||
}
|
|
||||||
hdfsWriteClient = new HoodieWriteClient(jsc, config);
|
|
||||||
return hdfsWriteClient;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
protected HoodieWriteConfig getHoodieWriteConfig(String basePath) {
|
protected HoodieWriteConfig getHoodieWriteConfig(String basePath) {
|
||||||
|
|||||||
@@ -76,7 +76,6 @@ public class TestHbaseIndex extends HoodieClientTestHarness {
|
|||||||
private static HBaseTestingUtility utility;
|
private static HBaseTestingUtility utility;
|
||||||
private static Configuration hbaseConfig;
|
private static Configuration hbaseConfig;
|
||||||
private static String tableName = "test_table";
|
private static String tableName = "test_table";
|
||||||
private HoodieWriteClient writeClient;
|
|
||||||
|
|
||||||
public TestHbaseIndex() throws Exception {
|
public TestHbaseIndex() throws Exception {
|
||||||
}
|
}
|
||||||
@@ -112,22 +111,13 @@ public class TestHbaseIndex extends HoodieClientTestHarness {
|
|||||||
|
|
||||||
@After
|
@After
|
||||||
public void tearDown() throws Exception {
|
public void tearDown() throws Exception {
|
||||||
if (null != writeClient) {
|
|
||||||
writeClient.close();
|
|
||||||
writeClient = null;
|
|
||||||
}
|
|
||||||
|
|
||||||
cleanupSparkContexts();
|
cleanupSparkContexts();
|
||||||
cleanupTempFolderAndPath();
|
cleanupTempFolderAndPath();
|
||||||
cleanupTestDataGenerator();
|
cleanupTestDataGenerator();
|
||||||
}
|
}
|
||||||
|
|
||||||
private HoodieWriteClient getWriteClient(HoodieWriteConfig config) throws Exception {
|
private HoodieWriteClient getWriteClient(HoodieWriteConfig config) throws Exception {
|
||||||
if (null != writeClient) {
|
return new HoodieWriteClient(jsc, config);
|
||||||
writeClient.close();
|
|
||||||
}
|
|
||||||
writeClient = new HoodieWriteClient(jsc, config);
|
|
||||||
return writeClient;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
|
|||||||
@@ -58,7 +58,6 @@ public class TestHoodieCommitArchiveLog extends HoodieClientTestHarness {
|
|||||||
@Before
|
@Before
|
||||||
public void init() throws Exception {
|
public void init() throws Exception {
|
||||||
initDFS();
|
initDFS();
|
||||||
|
|
||||||
initTempFolderAndPath();
|
initTempFolderAndPath();
|
||||||
initSparkContexts("TestHoodieCommitArchiveLog");
|
initSparkContexts("TestHoodieCommitArchiveLog");
|
||||||
hadoopConf = dfs.getConf();
|
hadoopConf = dfs.getConf();
|
||||||
@@ -70,7 +69,6 @@ public class TestHoodieCommitArchiveLog extends HoodieClientTestHarness {
|
|||||||
@After
|
@After
|
||||||
public void clean() throws IOException {
|
public void clean() throws IOException {
|
||||||
cleanupDFS();
|
cleanupDFS();
|
||||||
|
|
||||||
cleanupTempFolderAndPath();
|
cleanupTempFolderAndPath();
|
||||||
cleanupSparkContexts();
|
cleanupSparkContexts();
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -51,7 +51,6 @@ import org.junit.Test;
|
|||||||
|
|
||||||
public class TestHoodieCompactor extends HoodieClientTestHarness {
|
public class TestHoodieCompactor extends HoodieClientTestHarness {
|
||||||
|
|
||||||
private HoodieWriteClient writeClient;
|
|
||||||
private Configuration hadoopConf;
|
private Configuration hadoopConf;
|
||||||
|
|
||||||
@Before
|
@Before
|
||||||
@@ -76,11 +75,7 @@ public class TestHoodieCompactor extends HoodieClientTestHarness {
|
|||||||
}
|
}
|
||||||
|
|
||||||
private HoodieWriteClient getWriteClient(HoodieWriteConfig config) throws Exception {
|
private HoodieWriteClient getWriteClient(HoodieWriteConfig config) throws Exception {
|
||||||
if (null != writeClient) {
|
return new HoodieWriteClient(jsc, config);
|
||||||
writeClient.close();
|
|
||||||
}
|
|
||||||
writeClient = new HoodieWriteClient(jsc, config);
|
|
||||||
return writeClient;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private HoodieWriteConfig getConfig() {
|
private HoodieWriteConfig getConfig() {
|
||||||
|
|||||||
@@ -53,8 +53,6 @@ import org.junit.Test;
|
|||||||
@SuppressWarnings("unchecked")
|
@SuppressWarnings("unchecked")
|
||||||
public class TestHoodieMergeHandle extends HoodieClientTestHarness {
|
public class TestHoodieMergeHandle extends HoodieClientTestHarness {
|
||||||
|
|
||||||
private HoodieWriteClient writeClient;
|
|
||||||
|
|
||||||
@Before
|
@Before
|
||||||
public void setUp() throws Exception {
|
public void setUp() throws Exception {
|
||||||
initSparkContexts("TestHoodieMergeHandle");
|
initSparkContexts("TestHoodieMergeHandle");
|
||||||
@@ -74,11 +72,7 @@ public class TestHoodieMergeHandle extends HoodieClientTestHarness {
|
|||||||
}
|
}
|
||||||
|
|
||||||
private HoodieWriteClient getWriteClient(HoodieWriteConfig config) throws Exception {
|
private HoodieWriteClient getWriteClient(HoodieWriteConfig config) throws Exception {
|
||||||
if (null != writeClient) {
|
return new HoodieWriteClient(jsc, config);
|
||||||
writeClient.close();
|
|
||||||
}
|
|
||||||
writeClient = new HoodieWriteClient(jsc, config);
|
|
||||||
return writeClient;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
|
|||||||
@@ -78,8 +78,6 @@ import org.junit.rules.TemporaryFolder;
|
|||||||
|
|
||||||
public class TestMergeOnReadTable extends HoodieClientTestHarness {
|
public class TestMergeOnReadTable extends HoodieClientTestHarness {
|
||||||
|
|
||||||
private HoodieWriteClient writeClient;
|
|
||||||
|
|
||||||
@Before
|
@Before
|
||||||
public void init() throws IOException {
|
public void init() throws IOException {
|
||||||
initDFS();
|
initDFS();
|
||||||
@@ -100,11 +98,7 @@ public class TestMergeOnReadTable extends HoodieClientTestHarness {
|
|||||||
}
|
}
|
||||||
|
|
||||||
private HoodieWriteClient getWriteClient(HoodieWriteConfig config) throws Exception {
|
private HoodieWriteClient getWriteClient(HoodieWriteConfig config) throws Exception {
|
||||||
if (null != writeClient) {
|
return new HoodieWriteClient(jsc, config);
|
||||||
writeClient.close();
|
|
||||||
}
|
|
||||||
writeClient = new HoodieWriteClient(jsc, config);
|
|
||||||
return writeClient;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
|
|||||||
Reference in New Issue
Block a user