Reformatting code per Google Code Style all over
This commit is contained in:
Vinoth Chandar
committed by
vinoth chandar
vinoth chandar
parent
5a62480a92
commit
e45679f5e2
@@ -24,395 +24,401 @@ import com.uber.hoodie.common.util.ReflectionUtils;
|
||||
import com.uber.hoodie.index.HoodieIndex;
|
||||
import com.uber.hoodie.io.compact.strategy.CompactionStrategy;
|
||||
import com.uber.hoodie.metrics.MetricsReporterType;
|
||||
import org.apache.spark.storage.StorageLevel;
|
||||
|
||||
import javax.annotation.concurrent.Immutable;
|
||||
import java.io.File;
|
||||
import java.io.FileReader;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.util.Map;
|
||||
import java.util.Properties;
|
||||
import javax.annotation.concurrent.Immutable;
|
||||
import org.apache.spark.storage.StorageLevel;
|
||||
|
||||
/**
|
||||
* Class storing configs for the {@link com.uber.hoodie.HoodieWriteClient}
|
||||
*/
|
||||
@Immutable
|
||||
public class HoodieWriteConfig extends DefaultHoodieConfig {
|
||||
private static final String BASE_PATH_PROP = "hoodie.base.path";
|
||||
private static final String AVRO_SCHEMA = "hoodie.avro.schema";
|
||||
public static final String TABLE_NAME = "hoodie.table.name";
|
||||
private static final String DEFAULT_PARALLELISM = "200";
|
||||
private static final String INSERT_PARALLELISM = "hoodie.insert.shuffle.parallelism";
|
||||
private static final String BULKINSERT_PARALLELISM = "hoodie.bulkinsert.shuffle.parallelism";
|
||||
private static final String UPSERT_PARALLELISM = "hoodie.upsert.shuffle.parallelism";
|
||||
private static final String COMBINE_BEFORE_INSERT_PROP = "hoodie.combine.before.insert";
|
||||
private static final String DEFAULT_COMBINE_BEFORE_INSERT = "false";
|
||||
private static final String COMBINE_BEFORE_UPSERT_PROP = "hoodie.combine.before.upsert";
|
||||
private static final String DEFAULT_COMBINE_BEFORE_UPSERT = "true";
|
||||
private static final String WRITE_STATUS_STORAGE_LEVEL = "hoodie.write.status.storage.level";
|
||||
private static final String DEFAULT_WRITE_STATUS_STORAGE_LEVEL = "MEMORY_AND_DISK_SER";
|
||||
private static final String HOODIE_AUTO_COMMIT_PROP = "hoodie.auto.commit";
|
||||
private static final String DEFAULT_HOODIE_AUTO_COMMIT = "true";
|
||||
private static final String HOODIE_ASSUME_DATE_PARTITIONING_PROP = "hoodie.assume.date.partitioning";
|
||||
private static final String DEFAULT_ASSUME_DATE_PARTITIONING = "false";
|
||||
private static final String HOODIE_WRITE_STATUS_CLASS_PROP = "hoodie.writestatus.class";
|
||||
private static final String DEFAULT_HOODIE_WRITE_STATUS_CLASS = WriteStatus.class.getName();
|
||||
|
||||
private HoodieWriteConfig(Properties props) {
|
||||
super(props);
|
||||
}
|
||||
private static final String BASE_PATH_PROP = "hoodie.base.path";
|
||||
private static final String AVRO_SCHEMA = "hoodie.avro.schema";
|
||||
public static final String TABLE_NAME = "hoodie.table.name";
|
||||
private static final String DEFAULT_PARALLELISM = "200";
|
||||
private static final String INSERT_PARALLELISM = "hoodie.insert.shuffle.parallelism";
|
||||
private static final String BULKINSERT_PARALLELISM = "hoodie.bulkinsert.shuffle.parallelism";
|
||||
private static final String UPSERT_PARALLELISM = "hoodie.upsert.shuffle.parallelism";
|
||||
private static final String COMBINE_BEFORE_INSERT_PROP = "hoodie.combine.before.insert";
|
||||
private static final String DEFAULT_COMBINE_BEFORE_INSERT = "false";
|
||||
private static final String COMBINE_BEFORE_UPSERT_PROP = "hoodie.combine.before.upsert";
|
||||
private static final String DEFAULT_COMBINE_BEFORE_UPSERT = "true";
|
||||
private static final String WRITE_STATUS_STORAGE_LEVEL = "hoodie.write.status.storage.level";
|
||||
private static final String DEFAULT_WRITE_STATUS_STORAGE_LEVEL = "MEMORY_AND_DISK_SER";
|
||||
private static final String HOODIE_AUTO_COMMIT_PROP = "hoodie.auto.commit";
|
||||
private static final String DEFAULT_HOODIE_AUTO_COMMIT = "true";
|
||||
private static final String HOODIE_ASSUME_DATE_PARTITIONING_PROP = "hoodie.assume.date.partitioning";
|
||||
private static final String DEFAULT_ASSUME_DATE_PARTITIONING = "false";
|
||||
private static final String HOODIE_WRITE_STATUS_CLASS_PROP = "hoodie.writestatus.class";
|
||||
private static final String DEFAULT_HOODIE_WRITE_STATUS_CLASS = WriteStatus.class.getName();
|
||||
|
||||
/**
|
||||
* base properties
|
||||
**/
|
||||
public String getBasePath() {
|
||||
return props.getProperty(BASE_PATH_PROP);
|
||||
}
|
||||
private HoodieWriteConfig(Properties props) {
|
||||
super(props);
|
||||
}
|
||||
|
||||
public String getSchema() {
|
||||
return props.getProperty(AVRO_SCHEMA);
|
||||
}
|
||||
/**
|
||||
* base properties
|
||||
**/
|
||||
public String getBasePath() {
|
||||
return props.getProperty(BASE_PATH_PROP);
|
||||
}
|
||||
|
||||
public String getTableName() {
|
||||
return props.getProperty(TABLE_NAME);
|
||||
}
|
||||
public String getSchema() {
|
||||
return props.getProperty(AVRO_SCHEMA);
|
||||
}
|
||||
|
||||
public Boolean shouldAutoCommit() {
|
||||
return Boolean.parseBoolean(props.getProperty(HOODIE_AUTO_COMMIT_PROP));
|
||||
}
|
||||
public String getTableName() {
|
||||
return props.getProperty(TABLE_NAME);
|
||||
}
|
||||
|
||||
public Boolean shouldAssumeDatePartitioning() {
|
||||
return Boolean.parseBoolean(props.getProperty(HOODIE_ASSUME_DATE_PARTITIONING_PROP));
|
||||
}
|
||||
public Boolean shouldAutoCommit() {
|
||||
return Boolean.parseBoolean(props.getProperty(HOODIE_AUTO_COMMIT_PROP));
|
||||
}
|
||||
|
||||
public int getBulkInsertShuffleParallelism() {
|
||||
return Integer.parseInt(props.getProperty(BULKINSERT_PARALLELISM));
|
||||
}
|
||||
public Boolean shouldAssumeDatePartitioning() {
|
||||
return Boolean.parseBoolean(props.getProperty(HOODIE_ASSUME_DATE_PARTITIONING_PROP));
|
||||
}
|
||||
|
||||
public int getInsertShuffleParallelism() {
|
||||
return Integer.parseInt(props.getProperty(INSERT_PARALLELISM));
|
||||
}
|
||||
public int getBulkInsertShuffleParallelism() {
|
||||
return Integer.parseInt(props.getProperty(BULKINSERT_PARALLELISM));
|
||||
}
|
||||
|
||||
public int getUpsertShuffleParallelism() {
|
||||
return Integer.parseInt(props.getProperty(UPSERT_PARALLELISM));
|
||||
}
|
||||
public int getInsertShuffleParallelism() {
|
||||
return Integer.parseInt(props.getProperty(INSERT_PARALLELISM));
|
||||
}
|
||||
|
||||
public boolean shouldCombineBeforeInsert() {
|
||||
return Boolean.parseBoolean(props.getProperty(COMBINE_BEFORE_INSERT_PROP));
|
||||
}
|
||||
public int getUpsertShuffleParallelism() {
|
||||
return Integer.parseInt(props.getProperty(UPSERT_PARALLELISM));
|
||||
}
|
||||
|
||||
public boolean shouldCombineBeforeUpsert() {
|
||||
return Boolean.parseBoolean(props.getProperty(COMBINE_BEFORE_UPSERT_PROP));
|
||||
}
|
||||
public boolean shouldCombineBeforeInsert() {
|
||||
return Boolean.parseBoolean(props.getProperty(COMBINE_BEFORE_INSERT_PROP));
|
||||
}
|
||||
|
||||
public StorageLevel getWriteStatusStorageLevel() {
|
||||
return StorageLevel.fromString(props.getProperty(WRITE_STATUS_STORAGE_LEVEL));
|
||||
}
|
||||
public boolean shouldCombineBeforeUpsert() {
|
||||
return Boolean.parseBoolean(props.getProperty(COMBINE_BEFORE_UPSERT_PROP));
|
||||
}
|
||||
|
||||
public String getWriteStatusClassName() {
|
||||
return props.getProperty(HOODIE_WRITE_STATUS_CLASS_PROP);
|
||||
}
|
||||
public StorageLevel getWriteStatusStorageLevel() {
|
||||
return StorageLevel.fromString(props.getProperty(WRITE_STATUS_STORAGE_LEVEL));
|
||||
}
|
||||
|
||||
/**
|
||||
* compaction properties
|
||||
**/
|
||||
public HoodieCleaningPolicy getCleanerPolicy() {
|
||||
return HoodieCleaningPolicy
|
||||
.valueOf(props.getProperty(HoodieCompactionConfig.CLEANER_POLICY_PROP));
|
||||
}
|
||||
public String getWriteStatusClassName() {
|
||||
return props.getProperty(HOODIE_WRITE_STATUS_CLASS_PROP);
|
||||
}
|
||||
|
||||
public int getCleanerFileVersionsRetained() {
|
||||
return Integer.parseInt(
|
||||
props.getProperty(HoodieCompactionConfig.CLEANER_FILE_VERSIONS_RETAINED_PROP));
|
||||
}
|
||||
/**
|
||||
* compaction properties
|
||||
**/
|
||||
public HoodieCleaningPolicy getCleanerPolicy() {
|
||||
return HoodieCleaningPolicy
|
||||
.valueOf(props.getProperty(HoodieCompactionConfig.CLEANER_POLICY_PROP));
|
||||
}
|
||||
|
||||
public int getCleanerCommitsRetained() {
|
||||
return Integer
|
||||
.parseInt(props.getProperty(HoodieCompactionConfig.CLEANER_COMMITS_RETAINED_PROP));
|
||||
}
|
||||
public int getCleanerFileVersionsRetained() {
|
||||
return Integer.parseInt(
|
||||
props.getProperty(HoodieCompactionConfig.CLEANER_FILE_VERSIONS_RETAINED_PROP));
|
||||
}
|
||||
|
||||
public int getMaxCommitsToKeep() {
|
||||
return Integer.parseInt(props.getProperty(HoodieCompactionConfig.MAX_COMMITS_TO_KEEP));
|
||||
}
|
||||
public int getCleanerCommitsRetained() {
|
||||
return Integer
|
||||
.parseInt(props.getProperty(HoodieCompactionConfig.CLEANER_COMMITS_RETAINED_PROP));
|
||||
}
|
||||
|
||||
public int getMinCommitsToKeep() {
|
||||
return Integer.parseInt(props.getProperty(HoodieCompactionConfig.MIN_COMMITS_TO_KEEP));
|
||||
}
|
||||
public int getMaxCommitsToKeep() {
|
||||
return Integer.parseInt(props.getProperty(HoodieCompactionConfig.MAX_COMMITS_TO_KEEP));
|
||||
}
|
||||
|
||||
public int getParquetSmallFileLimit() {
|
||||
return Integer.parseInt(props.getProperty(HoodieCompactionConfig.PARQUET_SMALL_FILE_LIMIT_BYTES));
|
||||
}
|
||||
public int getMinCommitsToKeep() {
|
||||
return Integer.parseInt(props.getProperty(HoodieCompactionConfig.MIN_COMMITS_TO_KEEP));
|
||||
}
|
||||
|
||||
public int getCopyOnWriteInsertSplitSize() {
|
||||
return Integer.parseInt(
|
||||
props.getProperty(HoodieCompactionConfig.COPY_ON_WRITE_TABLE_INSERT_SPLIT_SIZE));
|
||||
}
|
||||
public int getParquetSmallFileLimit() {
|
||||
return Integer
|
||||
.parseInt(props.getProperty(HoodieCompactionConfig.PARQUET_SMALL_FILE_LIMIT_BYTES));
|
||||
}
|
||||
|
||||
public int getCopyOnWriteRecordSizeEstimate() {
|
||||
return Integer.parseInt(
|
||||
props.getProperty(HoodieCompactionConfig.COPY_ON_WRITE_TABLE_RECORD_SIZE_ESTIMATE));
|
||||
}
|
||||
public int getCopyOnWriteInsertSplitSize() {
|
||||
return Integer.parseInt(
|
||||
props.getProperty(HoodieCompactionConfig.COPY_ON_WRITE_TABLE_INSERT_SPLIT_SIZE));
|
||||
}
|
||||
|
||||
public boolean shouldAutoTuneInsertSplits() {
|
||||
return Boolean.parseBoolean(
|
||||
props.getProperty(HoodieCompactionConfig.COPY_ON_WRITE_TABLE_AUTO_SPLIT_INSERTS));
|
||||
}
|
||||
public int getCopyOnWriteRecordSizeEstimate() {
|
||||
return Integer.parseInt(
|
||||
props.getProperty(HoodieCompactionConfig.COPY_ON_WRITE_TABLE_RECORD_SIZE_ESTIMATE));
|
||||
}
|
||||
|
||||
public int getCleanerParallelism() {
|
||||
return Integer.parseInt(props.getProperty(HoodieCompactionConfig.CLEANER_PARALLELISM));
|
||||
}
|
||||
public boolean shouldAutoTuneInsertSplits() {
|
||||
return Boolean.parseBoolean(
|
||||
props.getProperty(HoodieCompactionConfig.COPY_ON_WRITE_TABLE_AUTO_SPLIT_INSERTS));
|
||||
}
|
||||
|
||||
public boolean isAutoClean() {
|
||||
return Boolean.parseBoolean(props.getProperty(HoodieCompactionConfig.AUTO_CLEAN_PROP));
|
||||
}
|
||||
public int getCleanerParallelism() {
|
||||
return Integer.parseInt(props.getProperty(HoodieCompactionConfig.CLEANER_PARALLELISM));
|
||||
}
|
||||
|
||||
public boolean isInlineCompaction() {
|
||||
return Boolean.parseBoolean(props.getProperty(HoodieCompactionConfig.INLINE_COMPACT_PROP));
|
||||
}
|
||||
public boolean isAutoClean() {
|
||||
return Boolean.parseBoolean(props.getProperty(HoodieCompactionConfig.AUTO_CLEAN_PROP));
|
||||
}
|
||||
|
||||
public int getInlineCompactDeltaCommitMax() {
|
||||
return Integer.parseInt(
|
||||
props.getProperty(HoodieCompactionConfig.INLINE_COMPACT_NUM_DELTA_COMMITS_PROP));
|
||||
}
|
||||
public boolean isInlineCompaction() {
|
||||
return Boolean.parseBoolean(props.getProperty(HoodieCompactionConfig.INLINE_COMPACT_PROP));
|
||||
}
|
||||
|
||||
public CompactionStrategy getCompactionStrategy() {
|
||||
return ReflectionUtils.loadClass(props.getProperty(HoodieCompactionConfig.COMPACTION_STRATEGY_PROP));
|
||||
}
|
||||
public int getInlineCompactDeltaCommitMax() {
|
||||
return Integer.parseInt(
|
||||
props.getProperty(HoodieCompactionConfig.INLINE_COMPACT_NUM_DELTA_COMMITS_PROP));
|
||||
}
|
||||
|
||||
public Long getTargetIOPerCompactionInMB() {
|
||||
return Long.parseLong(props.getProperty(HoodieCompactionConfig.TARGET_IO_PER_COMPACTION_IN_MB_PROP));
|
||||
}
|
||||
public CompactionStrategy getCompactionStrategy() {
|
||||
return ReflectionUtils
|
||||
.loadClass(props.getProperty(HoodieCompactionConfig.COMPACTION_STRATEGY_PROP));
|
||||
}
|
||||
|
||||
/**
|
||||
* index properties
|
||||
**/
|
||||
public HoodieIndex.IndexType getIndexType() {
|
||||
return HoodieIndex.IndexType.valueOf(props.getProperty(HoodieIndexConfig.INDEX_TYPE_PROP));
|
||||
}
|
||||
public Long getTargetIOPerCompactionInMB() {
|
||||
return Long
|
||||
.parseLong(props.getProperty(HoodieCompactionConfig.TARGET_IO_PER_COMPACTION_IN_MB_PROP));
|
||||
}
|
||||
|
||||
public int getBloomFilterNumEntries() {
|
||||
return Integer.parseInt(props.getProperty(HoodieIndexConfig.BLOOM_FILTER_NUM_ENTRIES));
|
||||
}
|
||||
/**
|
||||
* index properties
|
||||
**/
|
||||
public HoodieIndex.IndexType getIndexType() {
|
||||
return HoodieIndex.IndexType.valueOf(props.getProperty(HoodieIndexConfig.INDEX_TYPE_PROP));
|
||||
}
|
||||
|
||||
public double getBloomFilterFPP() {
|
||||
return Double.parseDouble(props.getProperty(HoodieIndexConfig.BLOOM_FILTER_FPP));
|
||||
}
|
||||
public int getBloomFilterNumEntries() {
|
||||
return Integer.parseInt(props.getProperty(HoodieIndexConfig.BLOOM_FILTER_NUM_ENTRIES));
|
||||
}
|
||||
|
||||
public String getHbaseZkQuorum() {
|
||||
return props.getProperty(HoodieIndexConfig.HBASE_ZKQUORUM_PROP);
|
||||
}
|
||||
public double getBloomFilterFPP() {
|
||||
return Double.parseDouble(props.getProperty(HoodieIndexConfig.BLOOM_FILTER_FPP));
|
||||
}
|
||||
|
||||
public int getHbaseZkPort() {
|
||||
return Integer.parseInt(props.getProperty(HoodieIndexConfig.HBASE_ZKPORT_PROP));
|
||||
}
|
||||
public String getHbaseZkQuorum() {
|
||||
return props.getProperty(HoodieIndexConfig.HBASE_ZKQUORUM_PROP);
|
||||
}
|
||||
|
||||
public String getHbaseTableName() {
|
||||
return props.getProperty(HoodieIndexConfig.HBASE_TABLENAME_PROP);
|
||||
}
|
||||
public int getHbaseZkPort() {
|
||||
return Integer.parseInt(props.getProperty(HoodieIndexConfig.HBASE_ZKPORT_PROP));
|
||||
}
|
||||
|
||||
public int getBloomIndexParallelism() {
|
||||
return Integer.parseInt(props.getProperty(HoodieIndexConfig.BLOOM_INDEX_PARALLELISM_PROP));
|
||||
}
|
||||
public String getHbaseTableName() {
|
||||
return props.getProperty(HoodieIndexConfig.HBASE_TABLENAME_PROP);
|
||||
}
|
||||
|
||||
public boolean getBloomIndexPruneByRanges() {
|
||||
return Boolean.parseBoolean(props.getProperty(HoodieIndexConfig.BLOOM_INDEX_PRUNE_BY_RANGES_PROP));
|
||||
}
|
||||
public int getBloomIndexParallelism() {
|
||||
return Integer.parseInt(props.getProperty(HoodieIndexConfig.BLOOM_INDEX_PARALLELISM_PROP));
|
||||
}
|
||||
|
||||
public boolean getBloomIndexUseCaching() {
|
||||
return Boolean.parseBoolean(props.getProperty(HoodieIndexConfig.BLOOM_INDEX_USE_CACHING_PROP));
|
||||
}
|
||||
public boolean getBloomIndexPruneByRanges() {
|
||||
return Boolean
|
||||
.parseBoolean(props.getProperty(HoodieIndexConfig.BLOOM_INDEX_PRUNE_BY_RANGES_PROP));
|
||||
}
|
||||
|
||||
public int getNumBucketsPerPartition() {
|
||||
return Integer.parseInt(props.getProperty(HoodieIndexConfig.BUCKETED_INDEX_NUM_BUCKETS_PROP));
|
||||
}
|
||||
public boolean getBloomIndexUseCaching() {
|
||||
return Boolean.parseBoolean(props.getProperty(HoodieIndexConfig.BLOOM_INDEX_USE_CACHING_PROP));
|
||||
}
|
||||
|
||||
/**
|
||||
* storage properties
|
||||
**/
|
||||
public int getParquetMaxFileSize() {
|
||||
return Integer.parseInt(props.getProperty(HoodieStorageConfig.PARQUET_FILE_MAX_BYTES));
|
||||
}
|
||||
public int getNumBucketsPerPartition() {
|
||||
return Integer.parseInt(props.getProperty(HoodieIndexConfig.BUCKETED_INDEX_NUM_BUCKETS_PROP));
|
||||
}
|
||||
|
||||
public int getParquetBlockSize() {
|
||||
return Integer.parseInt(props.getProperty(HoodieStorageConfig.PARQUET_BLOCK_SIZE_BYTES));
|
||||
}
|
||||
/**
|
||||
* storage properties
|
||||
**/
|
||||
public int getParquetMaxFileSize() {
|
||||
return Integer.parseInt(props.getProperty(HoodieStorageConfig.PARQUET_FILE_MAX_BYTES));
|
||||
}
|
||||
|
||||
public int getParquetPageSize() {
|
||||
return Integer.parseInt(props.getProperty(HoodieStorageConfig.PARQUET_PAGE_SIZE_BYTES));
|
||||
}
|
||||
public int getParquetBlockSize() {
|
||||
return Integer.parseInt(props.getProperty(HoodieStorageConfig.PARQUET_BLOCK_SIZE_BYTES));
|
||||
}
|
||||
|
||||
/**
|
||||
* metrics properties
|
||||
**/
|
||||
public boolean isMetricsOn() {
|
||||
return Boolean.parseBoolean(props.getProperty(HoodieMetricsConfig.METRICS_ON));
|
||||
}
|
||||
public int getParquetPageSize() {
|
||||
return Integer.parseInt(props.getProperty(HoodieStorageConfig.PARQUET_PAGE_SIZE_BYTES));
|
||||
}
|
||||
|
||||
public MetricsReporterType getMetricsReporterType() {
|
||||
return MetricsReporterType
|
||||
.valueOf(props.getProperty(HoodieMetricsConfig.METRICS_REPORTER_TYPE));
|
||||
}
|
||||
/**
|
||||
* metrics properties
|
||||
**/
|
||||
public boolean isMetricsOn() {
|
||||
return Boolean.parseBoolean(props.getProperty(HoodieMetricsConfig.METRICS_ON));
|
||||
}
|
||||
|
||||
public String getGraphiteServerHost() {
|
||||
return props.getProperty(HoodieMetricsConfig.GRAPHITE_SERVER_HOST);
|
||||
}
|
||||
public MetricsReporterType getMetricsReporterType() {
|
||||
return MetricsReporterType
|
||||
.valueOf(props.getProperty(HoodieMetricsConfig.METRICS_REPORTER_TYPE));
|
||||
}
|
||||
|
||||
public int getGraphiteServerPort() {
|
||||
return Integer.parseInt(props.getProperty(HoodieMetricsConfig.GRAPHITE_SERVER_PORT));
|
||||
}
|
||||
public String getGraphiteServerHost() {
|
||||
return props.getProperty(HoodieMetricsConfig.GRAPHITE_SERVER_HOST);
|
||||
}
|
||||
|
||||
public String getGraphiteMetricPrefix() {
|
||||
return props.getProperty(HoodieMetricsConfig.GRAPHITE_METRIC_PREFIX);
|
||||
}
|
||||
public int getGraphiteServerPort() {
|
||||
return Integer.parseInt(props.getProperty(HoodieMetricsConfig.GRAPHITE_SERVER_PORT));
|
||||
}
|
||||
|
||||
public static HoodieWriteConfig.Builder newBuilder() {
|
||||
return new Builder();
|
||||
}
|
||||
public String getGraphiteMetricPrefix() {
|
||||
return props.getProperty(HoodieMetricsConfig.GRAPHITE_METRIC_PREFIX);
|
||||
}
|
||||
|
||||
public static HoodieWriteConfig.Builder newBuilder() {
|
||||
return new Builder();
|
||||
}
|
||||
|
||||
|
||||
public static class Builder {
|
||||
private final Properties props = new Properties();
|
||||
private boolean isIndexConfigSet = false;
|
||||
private boolean isStorageConfigSet = false;
|
||||
private boolean isCompactionConfigSet = false;
|
||||
private boolean isMetricsConfigSet = false;
|
||||
private boolean isAutoCommit = true;
|
||||
|
||||
public Builder fromFile(File propertiesFile) throws IOException {
|
||||
FileReader reader = new FileReader(propertiesFile);
|
||||
try {
|
||||
this.props.load(reader);
|
||||
return this;
|
||||
} finally {
|
||||
reader.close();
|
||||
}
|
||||
}
|
||||
private final Properties props = new Properties();
|
||||
private boolean isIndexConfigSet = false;
|
||||
private boolean isStorageConfigSet = false;
|
||||
private boolean isCompactionConfigSet = false;
|
||||
private boolean isMetricsConfigSet = false;
|
||||
private boolean isAutoCommit = true;
|
||||
|
||||
public Builder fromInputStream(InputStream inputStream) throws IOException {
|
||||
try {
|
||||
this.props.load(inputStream);
|
||||
return this;
|
||||
} finally {
|
||||
inputStream.close();
|
||||
}
|
||||
}
|
||||
|
||||
public Builder withProps(Map kvprops) {
|
||||
props.putAll(kvprops);
|
||||
return this;
|
||||
}
|
||||
|
||||
public Builder withPath(String basePath) {
|
||||
props.setProperty(BASE_PATH_PROP, basePath);
|
||||
return this;
|
||||
}
|
||||
|
||||
public Builder withSchema(String schemaStr) {
|
||||
props.setProperty(AVRO_SCHEMA, schemaStr);
|
||||
return this;
|
||||
}
|
||||
|
||||
public Builder forTable(String tableName) {
|
||||
props.setProperty(TABLE_NAME, tableName);
|
||||
return this;
|
||||
}
|
||||
|
||||
public Builder withBulkInsertParallelism(int bulkInsertParallelism) {
|
||||
props.setProperty(BULKINSERT_PARALLELISM, String.valueOf(bulkInsertParallelism));
|
||||
return this;
|
||||
}
|
||||
|
||||
public Builder withParallelism(int insertShuffleParallelism, int upsertShuffleParallelism) {
|
||||
props.setProperty(INSERT_PARALLELISM, String.valueOf(insertShuffleParallelism));
|
||||
props.setProperty(UPSERT_PARALLELISM, String.valueOf(upsertShuffleParallelism));
|
||||
return this;
|
||||
}
|
||||
|
||||
public Builder combineInput(boolean onInsert, boolean onUpsert) {
|
||||
props.setProperty(COMBINE_BEFORE_INSERT_PROP, String.valueOf(onInsert));
|
||||
props.setProperty(COMBINE_BEFORE_UPSERT_PROP, String.valueOf(onUpsert));
|
||||
return this;
|
||||
}
|
||||
|
||||
public Builder withWriteStatusStorageLevel(String level) {
|
||||
props.setProperty(WRITE_STATUS_STORAGE_LEVEL, level);
|
||||
return this;
|
||||
}
|
||||
|
||||
public Builder withIndexConfig(HoodieIndexConfig indexConfig) {
|
||||
props.putAll(indexConfig.getProps());
|
||||
isIndexConfigSet = true;
|
||||
return this;
|
||||
}
|
||||
|
||||
public Builder withStorageConfig(HoodieStorageConfig storageConfig) {
|
||||
props.putAll(storageConfig.getProps());
|
||||
isStorageConfigSet = true;
|
||||
return this;
|
||||
}
|
||||
|
||||
public Builder withCompactionConfig(HoodieCompactionConfig compactionConfig) {
|
||||
props.putAll(compactionConfig.getProps());
|
||||
isCompactionConfigSet = true;
|
||||
return this;
|
||||
}
|
||||
|
||||
public Builder withMetricsConfig(HoodieMetricsConfig metricsConfig) {
|
||||
props.putAll(metricsConfig.getProps());
|
||||
isMetricsConfigSet = true;
|
||||
return this;
|
||||
}
|
||||
|
||||
public Builder withAutoCommit(boolean autoCommit) {
|
||||
props.setProperty(HOODIE_AUTO_COMMIT_PROP, String.valueOf(autoCommit));
|
||||
return this;
|
||||
}
|
||||
|
||||
public Builder withAssumeDatePartitioning(boolean assumeDatePartitioning) {
|
||||
props.setProperty(HOODIE_ASSUME_DATE_PARTITIONING_PROP, String.valueOf(assumeDatePartitioning));
|
||||
return this;
|
||||
}
|
||||
|
||||
public Builder withWriteStatusClass(Class<? extends WriteStatus> writeStatusClass) {
|
||||
props.setProperty(HOODIE_WRITE_STATUS_CLASS_PROP, writeStatusClass.getName());
|
||||
return this;
|
||||
}
|
||||
|
||||
public HoodieWriteConfig build() {
|
||||
HoodieWriteConfig config = new HoodieWriteConfig(props);
|
||||
// Check for mandatory properties
|
||||
Preconditions.checkArgument(config.getBasePath() != null);
|
||||
setDefaultOnCondition(props, !props.containsKey(INSERT_PARALLELISM), INSERT_PARALLELISM,
|
||||
DEFAULT_PARALLELISM);
|
||||
setDefaultOnCondition(props, !props.containsKey(BULKINSERT_PARALLELISM), BULKINSERT_PARALLELISM,
|
||||
DEFAULT_PARALLELISM);
|
||||
setDefaultOnCondition(props, !props.containsKey(UPSERT_PARALLELISM), UPSERT_PARALLELISM,
|
||||
DEFAULT_PARALLELISM);
|
||||
setDefaultOnCondition(props, !props.containsKey(COMBINE_BEFORE_INSERT_PROP),
|
||||
COMBINE_BEFORE_INSERT_PROP, DEFAULT_COMBINE_BEFORE_INSERT);
|
||||
setDefaultOnCondition(props, !props.containsKey(COMBINE_BEFORE_UPSERT_PROP),
|
||||
COMBINE_BEFORE_UPSERT_PROP, DEFAULT_COMBINE_BEFORE_UPSERT);
|
||||
setDefaultOnCondition(props, !props.containsKey(WRITE_STATUS_STORAGE_LEVEL),
|
||||
WRITE_STATUS_STORAGE_LEVEL, DEFAULT_WRITE_STATUS_STORAGE_LEVEL);
|
||||
setDefaultOnCondition(props, !props.containsKey(HOODIE_AUTO_COMMIT_PROP),
|
||||
HOODIE_AUTO_COMMIT_PROP, DEFAULT_HOODIE_AUTO_COMMIT);
|
||||
setDefaultOnCondition(props, !props.containsKey(HOODIE_ASSUME_DATE_PARTITIONING_PROP),
|
||||
HOODIE_ASSUME_DATE_PARTITIONING_PROP, DEFAULT_ASSUME_DATE_PARTITIONING);
|
||||
setDefaultOnCondition(props, !props.containsKey(HOODIE_WRITE_STATUS_CLASS_PROP),
|
||||
HOODIE_WRITE_STATUS_CLASS_PROP, DEFAULT_HOODIE_WRITE_STATUS_CLASS);
|
||||
|
||||
// Make sure the props is propagated
|
||||
setDefaultOnCondition(props, !isIndexConfigSet,
|
||||
HoodieIndexConfig.newBuilder().fromProperties(props).build());
|
||||
setDefaultOnCondition(props, !isStorageConfigSet,
|
||||
HoodieStorageConfig.newBuilder().fromProperties(props).build());
|
||||
setDefaultOnCondition(props, !isCompactionConfigSet,
|
||||
HoodieCompactionConfig.newBuilder().fromProperties(props).build());
|
||||
setDefaultOnCondition(props, !isMetricsConfigSet,
|
||||
HoodieMetricsConfig.newBuilder().fromProperties(props).build());
|
||||
return config;
|
||||
}
|
||||
public Builder fromFile(File propertiesFile) throws IOException {
|
||||
FileReader reader = new FileReader(propertiesFile);
|
||||
try {
|
||||
this.props.load(reader);
|
||||
return this;
|
||||
} finally {
|
||||
reader.close();
|
||||
}
|
||||
}
|
||||
|
||||
public Builder fromInputStream(InputStream inputStream) throws IOException {
|
||||
try {
|
||||
this.props.load(inputStream);
|
||||
return this;
|
||||
} finally {
|
||||
inputStream.close();
|
||||
}
|
||||
}
|
||||
|
||||
public Builder withProps(Map kvprops) {
|
||||
props.putAll(kvprops);
|
||||
return this;
|
||||
}
|
||||
|
||||
public Builder withPath(String basePath) {
|
||||
props.setProperty(BASE_PATH_PROP, basePath);
|
||||
return this;
|
||||
}
|
||||
|
||||
public Builder withSchema(String schemaStr) {
|
||||
props.setProperty(AVRO_SCHEMA, schemaStr);
|
||||
return this;
|
||||
}
|
||||
|
||||
public Builder forTable(String tableName) {
|
||||
props.setProperty(TABLE_NAME, tableName);
|
||||
return this;
|
||||
}
|
||||
|
||||
public Builder withBulkInsertParallelism(int bulkInsertParallelism) {
|
||||
props.setProperty(BULKINSERT_PARALLELISM, String.valueOf(bulkInsertParallelism));
|
||||
return this;
|
||||
}
|
||||
|
||||
public Builder withParallelism(int insertShuffleParallelism, int upsertShuffleParallelism) {
|
||||
props.setProperty(INSERT_PARALLELISM, String.valueOf(insertShuffleParallelism));
|
||||
props.setProperty(UPSERT_PARALLELISM, String.valueOf(upsertShuffleParallelism));
|
||||
return this;
|
||||
}
|
||||
|
||||
public Builder combineInput(boolean onInsert, boolean onUpsert) {
|
||||
props.setProperty(COMBINE_BEFORE_INSERT_PROP, String.valueOf(onInsert));
|
||||
props.setProperty(COMBINE_BEFORE_UPSERT_PROP, String.valueOf(onUpsert));
|
||||
return this;
|
||||
}
|
||||
|
||||
public Builder withWriteStatusStorageLevel(String level) {
|
||||
props.setProperty(WRITE_STATUS_STORAGE_LEVEL, level);
|
||||
return this;
|
||||
}
|
||||
|
||||
public Builder withIndexConfig(HoodieIndexConfig indexConfig) {
|
||||
props.putAll(indexConfig.getProps());
|
||||
isIndexConfigSet = true;
|
||||
return this;
|
||||
}
|
||||
|
||||
public Builder withStorageConfig(HoodieStorageConfig storageConfig) {
|
||||
props.putAll(storageConfig.getProps());
|
||||
isStorageConfigSet = true;
|
||||
return this;
|
||||
}
|
||||
|
||||
public Builder withCompactionConfig(HoodieCompactionConfig compactionConfig) {
|
||||
props.putAll(compactionConfig.getProps());
|
||||
isCompactionConfigSet = true;
|
||||
return this;
|
||||
}
|
||||
|
||||
public Builder withMetricsConfig(HoodieMetricsConfig metricsConfig) {
|
||||
props.putAll(metricsConfig.getProps());
|
||||
isMetricsConfigSet = true;
|
||||
return this;
|
||||
}
|
||||
|
||||
public Builder withAutoCommit(boolean autoCommit) {
|
||||
props.setProperty(HOODIE_AUTO_COMMIT_PROP, String.valueOf(autoCommit));
|
||||
return this;
|
||||
}
|
||||
|
||||
public Builder withAssumeDatePartitioning(boolean assumeDatePartitioning) {
|
||||
props.setProperty(HOODIE_ASSUME_DATE_PARTITIONING_PROP,
|
||||
String.valueOf(assumeDatePartitioning));
|
||||
return this;
|
||||
}
|
||||
|
||||
public Builder withWriteStatusClass(Class<? extends WriteStatus> writeStatusClass) {
|
||||
props.setProperty(HOODIE_WRITE_STATUS_CLASS_PROP, writeStatusClass.getName());
|
||||
return this;
|
||||
}
|
||||
|
||||
public HoodieWriteConfig build() {
|
||||
HoodieWriteConfig config = new HoodieWriteConfig(props);
|
||||
// Check for mandatory properties
|
||||
Preconditions.checkArgument(config.getBasePath() != null);
|
||||
setDefaultOnCondition(props, !props.containsKey(INSERT_PARALLELISM), INSERT_PARALLELISM,
|
||||
DEFAULT_PARALLELISM);
|
||||
setDefaultOnCondition(props, !props.containsKey(BULKINSERT_PARALLELISM),
|
||||
BULKINSERT_PARALLELISM,
|
||||
DEFAULT_PARALLELISM);
|
||||
setDefaultOnCondition(props, !props.containsKey(UPSERT_PARALLELISM), UPSERT_PARALLELISM,
|
||||
DEFAULT_PARALLELISM);
|
||||
setDefaultOnCondition(props, !props.containsKey(COMBINE_BEFORE_INSERT_PROP),
|
||||
COMBINE_BEFORE_INSERT_PROP, DEFAULT_COMBINE_BEFORE_INSERT);
|
||||
setDefaultOnCondition(props, !props.containsKey(COMBINE_BEFORE_UPSERT_PROP),
|
||||
COMBINE_BEFORE_UPSERT_PROP, DEFAULT_COMBINE_BEFORE_UPSERT);
|
||||
setDefaultOnCondition(props, !props.containsKey(WRITE_STATUS_STORAGE_LEVEL),
|
||||
WRITE_STATUS_STORAGE_LEVEL, DEFAULT_WRITE_STATUS_STORAGE_LEVEL);
|
||||
setDefaultOnCondition(props, !props.containsKey(HOODIE_AUTO_COMMIT_PROP),
|
||||
HOODIE_AUTO_COMMIT_PROP, DEFAULT_HOODIE_AUTO_COMMIT);
|
||||
setDefaultOnCondition(props, !props.containsKey(HOODIE_ASSUME_DATE_PARTITIONING_PROP),
|
||||
HOODIE_ASSUME_DATE_PARTITIONING_PROP, DEFAULT_ASSUME_DATE_PARTITIONING);
|
||||
setDefaultOnCondition(props, !props.containsKey(HOODIE_WRITE_STATUS_CLASS_PROP),
|
||||
HOODIE_WRITE_STATUS_CLASS_PROP, DEFAULT_HOODIE_WRITE_STATUS_CLASS);
|
||||
|
||||
// Make sure the props is propagated
|
||||
setDefaultOnCondition(props, !isIndexConfigSet,
|
||||
HoodieIndexConfig.newBuilder().fromProperties(props).build());
|
||||
setDefaultOnCondition(props, !isStorageConfigSet,
|
||||
HoodieStorageConfig.newBuilder().fromProperties(props).build());
|
||||
setDefaultOnCondition(props, !isCompactionConfigSet,
|
||||
HoodieCompactionConfig.newBuilder().fromProperties(props).build());
|
||||
setDefaultOnCondition(props, !isMetricsConfigSet,
|
||||
HoodieMetricsConfig.newBuilder().fromProperties(props).build());
|
||||
return config;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user