From df2e0c760e7df0bd1b200867b3f0d2ca3a3f1fce Mon Sep 17 00:00:00 2001 From: sathyaprakashg Date: Wed, 10 Jun 2020 16:16:44 -0700 Subject: [PATCH] HUDI-942 Increase default value number of delta commits for inline compaction (#1664) Co-authored-by: Sathyaprakash Govindasamy --- docker/demo/compaction.commands | 2 +- .../java/org/apache/hudi/config/HoodieCompactionConfig.java | 2 +- .../java/org/apache/hudi/client/TestTableSchemaEvolution.java | 2 ++ hudi-spark/src/test/java/HoodieJavaApp.java | 3 +++ 4 files changed, 7 insertions(+), 2 deletions(-) diff --git a/docker/demo/compaction.commands b/docker/demo/compaction.commands index 9853a3551..9bb8eb82a 100644 --- a/docker/demo/compaction.commands +++ b/docker/demo/compaction.commands @@ -17,6 +17,6 @@ connect --path /user/hive/warehouse/stock_ticks_mor compactions show all -compaction schedule +compaction schedule --hoodieConfigs hoodie.compact.inline.max.delta.commits=1 compaction run --parallelism 2 --sparkMemory 1G --schemaFilePath /var/demo/config/schema.avsc --retry 1 diff --git a/hudi-client/src/main/java/org/apache/hudi/config/HoodieCompactionConfig.java b/hudi-client/src/main/java/org/apache/hudi/config/HoodieCompactionConfig.java index f89fc0638..4ec048533 100644 --- a/hudi-client/src/main/java/org/apache/hudi/config/HoodieCompactionConfig.java +++ b/hudi-client/src/main/java/org/apache/hudi/config/HoodieCompactionConfig.java @@ -103,7 +103,7 @@ public class HoodieCompactionConfig extends DefaultHoodieConfig { private static final String DEFAULT_AUTO_CLEAN = "true"; private static final String DEFAULT_INLINE_COMPACT = "false"; private static final String DEFAULT_INCREMENTAL_CLEANER = "true"; - private static final String DEFAULT_INLINE_COMPACT_NUM_DELTA_COMMITS = "1"; + private static final String DEFAULT_INLINE_COMPACT_NUM_DELTA_COMMITS = "5"; private static final String DEFAULT_CLEANER_FILE_VERSIONS_RETAINED = "3"; private static final String DEFAULT_CLEANER_COMMITS_RETAINED = "10"; private static final String DEFAULT_MAX_COMMITS_TO_KEEP = "30"; diff --git a/hudi-client/src/test/java/org/apache/hudi/client/TestTableSchemaEvolution.java b/hudi-client/src/test/java/org/apache/hudi/client/TestTableSchemaEvolution.java index 25e97c928..8e56b3106 100644 --- a/hudi-client/src/test/java/org/apache/hudi/client/TestTableSchemaEvolution.java +++ b/hudi-client/src/test/java/org/apache/hudi/client/TestTableSchemaEvolution.java @@ -26,6 +26,7 @@ import org.apache.hudi.common.table.HoodieTableMetaClient; import org.apache.hudi.common.table.TableSchemaResolver; import org.apache.hudi.common.table.timeline.HoodieTimeline; import org.apache.hudi.common.util.Option; +import org.apache.hudi.config.HoodieCompactionConfig; import org.apache.hudi.config.HoodieIndexConfig; import org.apache.hudi.config.HoodieWriteConfig; import org.apache.hudi.exception.HoodieInsertException; @@ -467,6 +468,7 @@ public class TestTableSchemaEvolution extends HoodieClientTestBase { private HoodieWriteConfig getWriteConfig(String schema) { return getConfigBuilder(schema) .withIndexConfig(HoodieIndexConfig.newBuilder().withIndexType(IndexType.INMEMORY).build()) + .withCompactionConfig(HoodieCompactionConfig.newBuilder().withMaxNumDeltaCommitsBeforeCompaction(1).build()) .withAvroSchemaValidate(true) .build(); } diff --git a/hudi-spark/src/test/java/HoodieJavaApp.java b/hudi-spark/src/test/java/HoodieJavaApp.java index a179c4203..93493a662 100644 --- a/hudi-spark/src/test/java/HoodieJavaApp.java +++ b/hudi-spark/src/test/java/HoodieJavaApp.java @@ -21,6 +21,7 @@ import org.apache.hudi.DataSourceWriteOptions; import org.apache.hudi.HoodieDataSourceHelpers; import org.apache.hudi.common.model.HoodieRecord; import org.apache.hudi.common.model.HoodieTableType; +import org.apache.hudi.config.HoodieCompactionConfig; import org.apache.hudi.config.HoodieWriteConfig; import org.apache.hudi.hive.MultiPartKeysValueExtractor; import org.apache.hudi.hive.NonPartitionedExtractor; @@ -173,6 +174,7 @@ public class HoodieJavaApp { .option(DataSourceWriteOptions.KEYGENERATOR_CLASS_OPT_KEY(), nonPartitionedTable ? NonpartitionedKeyGenerator.class.getCanonicalName() : SimpleKeyGenerator.class.getCanonicalName()) // Add Key Extractor + .option(HoodieCompactionConfig.INLINE_COMPACT_NUM_DELTA_COMMITS_PROP, "1") .option(HoodieWriteConfig.TABLE_NAME, tableName).mode(SaveMode.Append); updateHiveSyncConfig(writer); @@ -197,6 +199,7 @@ public class HoodieJavaApp { .option(DataSourceWriteOptions.KEYGENERATOR_CLASS_OPT_KEY(), nonPartitionedTable ? NonpartitionedKeyGenerator.class.getCanonicalName() : SimpleKeyGenerator.class.getCanonicalName()) // Add Key Extractor + .option(HoodieCompactionConfig.INLINE_COMPACT_NUM_DELTA_COMMITS_PROP, "1") .option(HoodieWriteConfig.TABLE_NAME, tableName).mode(SaveMode.Append); updateHiveSyncConfig(writer);