1
0

Write smaller sized multiple blocks to log file instead of a large one

- Use SizeEstimator to size number of records to write
	- Configurable block size
   	- Configurable log file size
This commit is contained in:
Nishith Agarwal
2018-02-01 12:36:12 -08:00
committed by vinoth chandar
parent eb3d0c470f
commit d495484399
4 changed files with 110 additions and 61 deletions

View File

@@ -277,6 +277,15 @@ public class HoodieWriteConfig extends DefaultHoodieConfig {
return Integer.parseInt(props.getProperty(HoodieStorageConfig.PARQUET_PAGE_SIZE_BYTES));
}
public int getLogFileDataBlockMaxSize() {
return Integer.parseInt(props.getProperty(HoodieStorageConfig.LOGFILE_DATA_BLOCK_SIZE_MAX_BYTES));
}
public int getLogFileMaxSize() {
return Integer.parseInt(props.getProperty(HoodieStorageConfig.LOGFILE_SIZE_MAX_BYTES));
}
/**
* metrics properties
**/