lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-2151] Part1 Setting default parallelism to 200 for some of write configs (#3948)

Browse Source
This commit is contained in:
Sivabalan Narayanan
2021-11-12 07:29:37 -05:00
committed by GitHub
parent bc511edc85
commit 4f217fe718
2 changed files with 6 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java
View File

@@ -159,12 +159,12 @@ public class HoodieWriteConfig extends HoodieConfig {
public static final ConfigProperty<String> INSERT_PARALLELISM_VALUE = ConfigProperty public static final ConfigProperty<String> INSERT_PARALLELISM_VALUE = ConfigProperty
.key("hoodie.insert.shuffle.parallelism") .key("hoodie.insert.shuffle.parallelism")
.defaultValue("1500") .defaultValue("200")
.withDocumentation("Parallelism for inserting records into the table. Inserts can shuffle data before writing to tune file sizes and optimize the storage layout."); .withDocumentation("Parallelism for inserting records into the table. Inserts can shuffle data before writing to tune file sizes and optimize the storage layout.");
public static final ConfigProperty<String> BULKINSERT_PARALLELISM_VALUE = ConfigProperty public static final ConfigProperty<String> BULKINSERT_PARALLELISM_VALUE = ConfigProperty
.key("hoodie.bulkinsert.shuffle.parallelism") .key("hoodie.bulkinsert.shuffle.parallelism")
.defaultValue("1500") .defaultValue("200")
.withDocumentation("For large initial imports using bulk_insert operation, controls the parallelism to use for sort modes or custom partitioning done" .withDocumentation("For large initial imports using bulk_insert operation, controls the parallelism to use for sort modes or custom partitioning done"
+ "before writing records to the table."); + "before writing records to the table.");
@@ -183,13 +183,13 @@ public class HoodieWriteConfig extends HoodieConfig {
public static final ConfigProperty<String> UPSERT_PARALLELISM_VALUE = ConfigProperty public static final ConfigProperty<String> UPSERT_PARALLELISM_VALUE = ConfigProperty
.key("hoodie.upsert.shuffle.parallelism") .key("hoodie.upsert.shuffle.parallelism")
.defaultValue("1500") .defaultValue("200")
.withDocumentation("Parallelism to use for upsert operation on the table. Upserts can shuffle data to perform index lookups, file sizing, bin packing records optimally" .withDocumentation("Parallelism to use for upsert operation on the table. Upserts can shuffle data to perform index lookups, file sizing, bin packing records optimally"
+ "into file groups."); + "into file groups.");
public static final ConfigProperty<String> DELETE_PARALLELISM_VALUE = ConfigProperty public static final ConfigProperty<String> DELETE_PARALLELISM_VALUE = ConfigProperty
.key("hoodie.delete.shuffle.parallelism") .key("hoodie.delete.shuffle.parallelism")
.defaultValue("1500") .defaultValue("200")
.withDocumentation("Parallelism used for “delete” operation. Delete operations also performs shuffles, similar to upsert operation."); .withDocumentation("Parallelism used for “delete” operation. Delete operations also performs shuffles, similar to upsert operation.");
public static final ConfigProperty<String> ROLLBACK_PARALLELISM_VALUE = ConfigProperty public static final ConfigProperty<String> ROLLBACK_PARALLELISM_VALUE = ConfigProperty
@@ -241,7 +241,7 @@ public class HoodieWriteConfig extends HoodieConfig {
public static final ConfigProperty<String> FINALIZE_WRITE_PARALLELISM_VALUE = ConfigProperty public static final ConfigProperty<String> FINALIZE_WRITE_PARALLELISM_VALUE = ConfigProperty
.key("hoodie.finalize.write.parallelism") .key("hoodie.finalize.write.parallelism")
.defaultValue("1500") .defaultValue("200")
.withDocumentation("Parallelism for the write finalization internal operation, which involves removing any partially written " .withDocumentation("Parallelism for the write finalization internal operation, which involves removing any partially written "
+ "files from lake storage, before committing the write. Reduce this value, if the high number of tasks incur delays for smaller tables " + "files from lake storage, before committing the write. Reduce this value, if the high number of tasks incur delays for smaller tables "
+ "or low latency writes."); + "or low latency writes.");

2
hudi-common/src/main/java/org/apache/hudi/common/config/HoodieMetadataConfig.java
View File

@@ -111,7 +111,7 @@ public final class HoodieMetadataConfig extends HoodieConfig {
public static final ConfigProperty<Integer> FILE_LISTING_PARALLELISM_VALUE = ConfigProperty public static final ConfigProperty<Integer> FILE_LISTING_PARALLELISM_VALUE = ConfigProperty
.key("hoodie.file.listing.parallelism") .key("hoodie.file.listing.parallelism")
.defaultValue(1500) .defaultValue(200)
.sinceVersion("0.7.0") .sinceVersion("0.7.0")
.withDocumentation("Parallelism to use, when listing the table on lake storage."); .withDocumentation("Parallelism to use, when listing the table on lake storage.");