Performing commit archiving in batches to avoid keeping a huge chunk in memory

2019-04-07 11:12:22 -07:00
parent b07110b9fd
commit a8feee9293
3 changed files with 28 additions and 4 deletions
--- a/hoodie-client/src/main/java/com/uber/hoodie/config/HoodieWriteConfig.java
+++ b/hoodie-client/src/main/java/com/uber/hoodie/config/HoodieWriteConfig.java
@@ -249,6 +249,11 @@ public class HoodieWriteConfig extends DefaultHoodieConfig {
        .parseInt(props.getProperty(HoodieCompactionConfig.TARGET_PARTITIONS_PER_DAYBASED_COMPACTION_PROP));
  }

+  public int getCommitArchivalBatchSize() {
+    return Integer
+        .parseInt(props.getProperty(HoodieCompactionConfig.COMMITS_ARCHIVAL_BATCH_SIZE_PROP));
+  }
+
  /**
   * index properties
   **/