lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

Make commit a public method. Introduce a auto-commit config. Relates issue https://github.com/uber/hoodie/issues/58 (#60)

Browse Source
This commit is contained in:
prazanna
2017-01-10 22:14:40 -08:00
committed by vinoth chandar
parent 9d09a58a18
commit e4e3395f3e
3 changed files with 67 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files

40
hoodie-client/src/test/java/com/uber/hoodie/TestHoodieClient.java
View File

@@ -148,6 +148,35 @@ public class TestHoodieClient implements Serializable {
assertTrue(result.size() == 25);
}
@Test
public void testAutoCommit() throws Exception {
// Set autoCommit false
HoodieWriteConfig cfg = getConfigBuilder().withAutoCommit(false).build();
HoodieWriteClient client = new HoodieWriteClient(jsc, cfg);
String newCommitTime = "001";
List<HoodieRecord> records = dataGen.generateInserts(newCommitTime, 200);
JavaRDD<HoodieRecord> writeRecords = jsc.parallelize(records, 1);
JavaRDD<WriteStatus> result = client.insert(writeRecords, newCommitTime);
assertFalse("If Autocommit is false, then commit should not be made automatically",
HoodieTestUtils.doesCommitExist(basePath, newCommitTime));
assertTrue("Commit should succeed", client.commit(newCommitTime, result));
assertTrue("After explicit commit, commit file should be created",
HoodieTestUtils.doesCommitExist(basePath, newCommitTime));
newCommitTime = "002";
records = dataGen.generateUpdates(newCommitTime, 100);
JavaRDD<HoodieRecord> updateRecords = jsc.parallelize(records, 1);
result = client.upsert(writeRecords, newCommitTime);
assertFalse("If Autocommit is false, then commit should not be made automatically",
HoodieTestUtils.doesCommitExist(basePath, newCommitTime));
assertTrue("Commit should succeed", client.commit(newCommitTime, result));
assertTrue("After explicit commit, commit file should be created",
HoodieTestUtils.doesCommitExist(basePath, newCommitTime));
}
@Test
public void testUpserts() throws Exception {
HoodieWriteConfig cfg = getConfig();
@@ -210,13 +239,12 @@ public class TestHoodieClient implements Serializable {
// Check that the incremental consumption from time 000
assertEquals("Incremental consumption from time 002, should give all records in commit 004",
readClient.readCommit(newCommitTime).count(),
readClient.readSince("002").count());
readClient.readCommit(newCommitTime).count(),
readClient.readSince("002").count());
assertEquals("Incremental consumption from time 001, should give all records in commit 004",
readClient.readCommit(newCommitTime).count(),
readClient.readSince("001").count());
readClient.readCommit(newCommitTime).count(),
readClient.readSince("001").count());
}
@Test
public void testInsertAndCleanByVersions() throws Exception {
int maxVersions = 2; // keep upto 2 versions for each file
@@ -513,6 +541,8 @@ public class TestHoodieClient implements Serializable {
HoodieTestUtils.doesDataFileExist(basePath, "2016/05/06", commitTime1, file13));
}
@Test
public void testSmallInsertHandling() throws Exception {