lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

Fixing small file handling, inline compaction defaults

Browse Source 
- Small file limit is now 100MB by default
 - Turned on inline compaction by default for MOR
 - Changes take effect on DataSource and DeltaStreamer
This commit is contained in:
Vinoth Chandar
2019-03-12 15:59:41 -07:00
committed by n3nash
parent 51f4908989
commit b34a204a52
5 changed files with 22 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
hoodie-client/src/main/java/com/uber/hoodie/config/HoodieCompactionConfig.java
View File

@@ -47,8 +47,8 @@ public class HoodieCompactionConfig extends DefaultHoodieConfig {
public static final String MIN_COMMITS_TO_KEEP_PROP = "hoodie.keep.min.commits";
// Upsert uses this file size to compact new data onto existing files..
public static final String PARQUET_SMALL_FILE_LIMIT_BYTES = "hoodie.parquet.small.file.limit";
// Turned off by default
public static final String DEFAULT_PARQUET_SMALL_FILE_LIMIT_BYTES = String.valueOf(0);
// By default, treat any file <= 100MB as a small file.
public static final String DEFAULT_PARQUET_SMALL_FILE_LIMIT_BYTES = String.valueOf(104857600);
/**
* Configs related to specific table types
**/