lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-3147] Add endpoint_url to dynamodb lock provider (#4500)

Browse Source 
Co-authored-by: Nicolas Paris <nicolas.paris@adevinta.com>
This commit is contained in:
Nicolas Paris
2022-01-04 22:42:28 +01:00
committed by GitHub
parent bf4e3d63e7
commit 37b15ff458
2 changed files with 17 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
hudi-aws/src/main/java/org/apache/hudi/aws/transaction/lock/DynamoDBBasedLockProvider.java
View File

@@ -156,7 +156,9 @@ public class DynamoDBBasedLockProvider implements LockProvider<LockItem> {
private AmazonDynamoDB getDynamoDBClient() { private AmazonDynamoDB getDynamoDBClient() {
String region = this.lockConfiguration.getConfig().getString(DynamoDbBasedLockConfig.DYNAMODB_LOCK_REGION.key()); String region = this.lockConfiguration.getConfig().getString(DynamoDbBasedLockConfig.DYNAMODB_LOCK_REGION.key());
String endpointURL = RegionUtils.getRegion(region).getServiceEndpoint(AmazonDynamoDB.ENDPOINT_PREFIX); String endpointURL = this.lockConfiguration.getConfig().getString(DynamoDbBasedLockConfig.DYNAMODB_ENDPOINT_URL.key()) == null
? RegionUtils.getRegion(region).getServiceEndpoint(AmazonDynamoDB.ENDPOINT_PREFIX)
: this.lockConfiguration.getConfig().getString(DynamoDbBasedLockConfig.DYNAMODB_ENDPOINT_URL.key());
AwsClientBuilder.EndpointConfiguration dynamodbEndpoint = AwsClientBuilder.EndpointConfiguration dynamodbEndpoint =
new AwsClientBuilder.EndpointConfiguration(endpointURL, region); new AwsClientBuilder.EndpointConfiguration(endpointURL, region);
return AmazonDynamoDBClientBuilder.standard() return AmazonDynamoDBClientBuilder.standard()

21
hudi-aws/src/main/java/org/apache/hudi/config/DynamoDbBasedLockConfig.java
View File

@@ -34,10 +34,10 @@ import static org.apache.hudi.common.config.LockConfiguration.LOCK_PREFIX;
* Hoodie Configs for Locks. * Hoodie Configs for Locks.
*/ */
@ConfigClassProperty(name = "DynamoDB based Locks Configurations", @ConfigClassProperty(name = "DynamoDB based Locks Configurations",
groupName = ConfigGroups.Names.WRITE_CLIENT, groupName = ConfigGroups.Names.WRITE_CLIENT,
description = "Configs that control DynamoDB based locking mechanisms required for concurrency control " description = "Configs that control DynamoDB based locking mechanisms required for concurrency control "
+ " between writers to a Hudi table. Concurrency between Hudi's own table services " + " between writers to a Hudi table. Concurrency between Hudi's own table services "
+ " are auto managed internally.") + " are auto managed internally.")
public class DynamoDbBasedLockConfig extends HoodieConfig { public class DynamoDbBasedLockConfig extends HoodieConfig {
// configs for DynamoDb based locks // configs for DynamoDb based locks
@@ -60,8 +60,8 @@ public class DynamoDbBasedLockConfig extends HoodieConfig {
return Option.empty(); return Option.empty();
}) })
.withDocumentation("For DynamoDB based lock provider, the partition key for the DynamoDB lock table. " .withDocumentation("For DynamoDB based lock provider, the partition key for the DynamoDB lock table. "
+ "Each Hudi dataset should has it's unique key so concurrent writers could refer to the same partition key." + "Each Hudi dataset should has it's unique key so concurrent writers could refer to the same partition key."
+ " By default we use the Hudi table name specified to be the partition key"); + " By default we use the Hudi table name specified to be the partition key");
public static final ConfigProperty<String> DYNAMODB_LOCK_REGION = ConfigProperty public static final ConfigProperty<String> DYNAMODB_LOCK_REGION = ConfigProperty
.key(DYNAMODB_BASED_LOCK_PROPERTY_PREFIX + "region") .key(DYNAMODB_BASED_LOCK_PROPERTY_PREFIX + "region")
@@ -75,7 +75,7 @@ public class DynamoDbBasedLockConfig extends HoodieConfig {
return Option.empty(); return Option.empty();
}) })
.withDocumentation("For DynamoDB based lock provider, the region used in endpoint for Amazon DynamoDB service." .withDocumentation("For DynamoDB based lock provider, the region used in endpoint for Amazon DynamoDB service."
+ " Would try to first get it from AWS_REGION environment variable. If not find, by default use us-east-1"); + " Would try to first get it from AWS_REGION environment variable. If not find, by default use us-east-1");
public static final ConfigProperty<String> DYNAMODB_LOCK_BILLING_MODE = ConfigProperty public static final ConfigProperty<String> DYNAMODB_LOCK_BILLING_MODE = ConfigProperty
.key(DYNAMODB_BASED_LOCK_PROPERTY_PREFIX + "billing_mode") .key(DYNAMODB_BASED_LOCK_PROPERTY_PREFIX + "billing_mode")
@@ -100,4 +100,11 @@ public class DynamoDbBasedLockConfig extends HoodieConfig {
.defaultValue(String.valueOf(10 * 60 * 1000)) .defaultValue(String.valueOf(10 * 60 * 1000))
.sinceVersion("0.10.0") .sinceVersion("0.10.0")
.withDocumentation("For DynamoDB based lock provider, the maximum number of milliseconds to wait for creating DynamoDB table"); .withDocumentation("For DynamoDB based lock provider, the maximum number of milliseconds to wait for creating DynamoDB table");
public static final ConfigProperty<String> DYNAMODB_ENDPOINT_URL = ConfigProperty
.key(DYNAMODB_BASED_LOCK_PROPERTY_PREFIX + "endpoint_url")
.defaultValue("us-east-1")
.sinceVersion("0.11.0")
.withDocumentation("For DynamoDB based lock provider, the url endpoint used for Amazon DynamoDB service."
+ " Useful for development with a local dynamodb instance.");
} }