1
0

Optimize hoodie client after implementat auto closeable interface

This commit is contained in:
yanghua
2019-09-03 19:00:37 +08:00
committed by vinoth chandar
parent 90bfb900aa
commit 80c27f2351
7 changed files with 7 additions and 62 deletions

View File

@@ -65,9 +65,6 @@ public class TestHoodieClientBase extends HoodieClientTestHarness {
protected static Logger logger = LogManager.getLogger(TestHoodieClientBase.class); protected static Logger logger = LogManager.getLogger(TestHoodieClientBase.class);
private HoodieWriteClient writeClient;
private HoodieReadClient readClient;
protected HoodieWriteClient getHoodieWriteClient(HoodieWriteConfig cfg) { protected HoodieWriteClient getHoodieWriteClient(HoodieWriteConfig cfg) {
return getHoodieWriteClient(cfg, false); return getHoodieWriteClient(cfg, false);
} }
@@ -78,29 +75,11 @@ public class TestHoodieClientBase extends HoodieClientTestHarness {
protected HoodieWriteClient getHoodieWriteClient(HoodieWriteConfig cfg, boolean rollbackInflightCommit, protected HoodieWriteClient getHoodieWriteClient(HoodieWriteConfig cfg, boolean rollbackInflightCommit,
HoodieIndex index) { HoodieIndex index) {
closeWriteClient(); return new HoodieWriteClient(jsc, cfg, rollbackInflightCommit, index);
writeClient = new HoodieWriteClient(jsc, cfg, rollbackInflightCommit, index);
return writeClient;
} }
protected HoodieReadClient getHoodieReadClient(String basePath) { protected HoodieReadClient getHoodieReadClient(String basePath) {
closeReadClient(); return new HoodieReadClient(jsc, basePath);
readClient = new HoodieReadClient(jsc, basePath);
return readClient;
}
private void closeWriteClient() {
if (null != writeClient) {
writeClient.close();
writeClient = null;
}
}
private void closeReadClient() {
if (null != readClient) {
readClient.close();
readClient = null;
}
} }
/** /**

View File

@@ -49,7 +49,6 @@ public class TestMultiFS extends HoodieClientTestHarness {
private static Logger logger = LogManager.getLogger(TestMultiFS.class); private static Logger logger = LogManager.getLogger(TestMultiFS.class);
private String tablePath = "file:///tmp/hoodie/sample-table"; private String tablePath = "file:///tmp/hoodie/sample-table";
protected String tableName = "hoodie_rt"; protected String tableName = "hoodie_rt";
private HoodieWriteClient hdfsWriteClient;
private String tableType = HoodieTableType.COPY_ON_WRITE.name(); private String tableType = HoodieTableType.COPY_ON_WRITE.name();
@Before @Before
@@ -67,11 +66,7 @@ public class TestMultiFS extends HoodieClientTestHarness {
} }
private HoodieWriteClient getHoodieWriteClient(HoodieWriteConfig config) throws Exception { private HoodieWriteClient getHoodieWriteClient(HoodieWriteConfig config) throws Exception {
if (null != hdfsWriteClient) { return new HoodieWriteClient(jsc, config);
hdfsWriteClient.close();
}
hdfsWriteClient = new HoodieWriteClient(jsc, config);
return hdfsWriteClient;
} }
protected HoodieWriteConfig getHoodieWriteConfig(String basePath) { protected HoodieWriteConfig getHoodieWriteConfig(String basePath) {

View File

@@ -76,7 +76,6 @@ public class TestHbaseIndex extends HoodieClientTestHarness {
private static HBaseTestingUtility utility; private static HBaseTestingUtility utility;
private static Configuration hbaseConfig; private static Configuration hbaseConfig;
private static String tableName = "test_table"; private static String tableName = "test_table";
private HoodieWriteClient writeClient;
public TestHbaseIndex() throws Exception { public TestHbaseIndex() throws Exception {
} }
@@ -112,22 +111,13 @@ public class TestHbaseIndex extends HoodieClientTestHarness {
@After @After
public void tearDown() throws Exception { public void tearDown() throws Exception {
if (null != writeClient) {
writeClient.close();
writeClient = null;
}
cleanupSparkContexts(); cleanupSparkContexts();
cleanupTempFolderAndPath(); cleanupTempFolderAndPath();
cleanupTestDataGenerator(); cleanupTestDataGenerator();
} }
private HoodieWriteClient getWriteClient(HoodieWriteConfig config) throws Exception { private HoodieWriteClient getWriteClient(HoodieWriteConfig config) throws Exception {
if (null != writeClient) { return new HoodieWriteClient(jsc, config);
writeClient.close();
}
writeClient = new HoodieWriteClient(jsc, config);
return writeClient;
} }
@Test @Test

View File

@@ -58,7 +58,6 @@ public class TestHoodieCommitArchiveLog extends HoodieClientTestHarness {
@Before @Before
public void init() throws Exception { public void init() throws Exception {
initDFS(); initDFS();
initTempFolderAndPath(); initTempFolderAndPath();
initSparkContexts("TestHoodieCommitArchiveLog"); initSparkContexts("TestHoodieCommitArchiveLog");
hadoopConf = dfs.getConf(); hadoopConf = dfs.getConf();
@@ -70,7 +69,6 @@ public class TestHoodieCommitArchiveLog extends HoodieClientTestHarness {
@After @After
public void clean() throws IOException { public void clean() throws IOException {
cleanupDFS(); cleanupDFS();
cleanupTempFolderAndPath(); cleanupTempFolderAndPath();
cleanupSparkContexts(); cleanupSparkContexts();
} }

View File

@@ -51,7 +51,6 @@ import org.junit.Test;
public class TestHoodieCompactor extends HoodieClientTestHarness { public class TestHoodieCompactor extends HoodieClientTestHarness {
private HoodieWriteClient writeClient;
private Configuration hadoopConf; private Configuration hadoopConf;
@Before @Before
@@ -76,11 +75,7 @@ public class TestHoodieCompactor extends HoodieClientTestHarness {
} }
private HoodieWriteClient getWriteClient(HoodieWriteConfig config) throws Exception { private HoodieWriteClient getWriteClient(HoodieWriteConfig config) throws Exception {
if (null != writeClient) { return new HoodieWriteClient(jsc, config);
writeClient.close();
}
writeClient = new HoodieWriteClient(jsc, config);
return writeClient;
} }
private HoodieWriteConfig getConfig() { private HoodieWriteConfig getConfig() {

View File

@@ -53,8 +53,6 @@ import org.junit.Test;
@SuppressWarnings("unchecked") @SuppressWarnings("unchecked")
public class TestHoodieMergeHandle extends HoodieClientTestHarness { public class TestHoodieMergeHandle extends HoodieClientTestHarness {
private HoodieWriteClient writeClient;
@Before @Before
public void setUp() throws Exception { public void setUp() throws Exception {
initSparkContexts("TestHoodieMergeHandle"); initSparkContexts("TestHoodieMergeHandle");
@@ -74,11 +72,7 @@ public class TestHoodieMergeHandle extends HoodieClientTestHarness {
} }
private HoodieWriteClient getWriteClient(HoodieWriteConfig config) throws Exception { private HoodieWriteClient getWriteClient(HoodieWriteConfig config) throws Exception {
if (null != writeClient) { return new HoodieWriteClient(jsc, config);
writeClient.close();
}
writeClient = new HoodieWriteClient(jsc, config);
return writeClient;
} }
@Test @Test

View File

@@ -78,8 +78,6 @@ import org.junit.rules.TemporaryFolder;
public class TestMergeOnReadTable extends HoodieClientTestHarness { public class TestMergeOnReadTable extends HoodieClientTestHarness {
private HoodieWriteClient writeClient;
@Before @Before
public void init() throws IOException { public void init() throws IOException {
initDFS(); initDFS();
@@ -100,11 +98,7 @@ public class TestMergeOnReadTable extends HoodieClientTestHarness {
} }
private HoodieWriteClient getWriteClient(HoodieWriteConfig config) throws Exception { private HoodieWriteClient getWriteClient(HoodieWriteConfig config) throws Exception {
if (null != writeClient) { return new HoodieWriteClient(jsc, config);
writeClient.close();
}
writeClient = new HoodieWriteClient(jsc, config);
return writeClient;
} }
@Test @Test