lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

Add config for index parallelism and make clean public (#109)

Browse Source 
* Add config for index parallelism and make clean public

* Review comments on clean api modification
This commit is contained in:
prazanna
2017-03-21 17:36:46 -07:00
committed by GitHub
parent 21898907c1
commit f1b7afad21
5 changed files with 51 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

21
hoodie-client/src/main/java/com/uber/hoodie/HoodieWriteClient.java
View File

@@ -367,8 +367,13 @@ public class HoodieWriteClient<T extends HoodieRecordPayload> implements Seriali
// Save was a success
// We cannot have unbounded commit files. Archive commits if we have to archive
archiveLog.archiveIfRequired();
// Call clean to cleanup if there is anything to cleanup after the commit,
clean(commitTime);
if(config.isAutoClean()) {
// Call clean to cleanup if there is anything to cleanup after the commit,
logger.info("Auto cleaning is enabled. Running cleaner now");
clean(commitTime);
} else {
logger.info("Auto cleaning is not enabled. Not running cleaner now");
}
if (writeContext != null) {
long durationInMs = metrics.getDurationInMs(writeContext.stop());
metrics.updateCommitMetrics(
@@ -698,6 +703,18 @@ public class HoodieWriteClient<T extends HoodieRecordPayload> implements Seriali
/**
* Clean up any stale/old files/data lying around (either on file storage or index storage)
* based on the configurations and CleaningPolicy used. (typically files that no longer can be used
* by a running query can be cleaned)
*/
public void clean() throws HoodieIOException {
String startCleanTime = startCommit();
clean(startCleanTime);
}
/**
* Clean up any stale/old files/data lying around (either on file storage or index storage)
* based on the configurations and CleaningPolicy used. (typically files that no longer can be used
* by a running query can be cleaned)
*/
private void clean(String startCleanTime) throws HoodieIOException {
try {