lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-1712] Rename & standardize config to match other configs (#2708)

Browse Source
This commit is contained in:
n3nash
2021-03-24 02:24:02 -07:00
committed by GitHub
parent 03668dbaf1
commit 01a1d7997b
3 changed files with 27 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
hudi-common/src/main/java/org/apache/hudi/common/config/LockConfiguration.java
View File

@@ -27,7 +27,7 @@ import java.util.Properties;
*/ */
public class LockConfiguration implements Serializable { public class LockConfiguration implements Serializable {
public static final String LOCK_PREFIX = "hoodie.writer.lock."; public static final String LOCK_PREFIX = "hoodie.write.lock.";
public static final String LOCK_ACQUIRE_RETRY_WAIT_TIME_IN_MILLIS_PROP = LOCK_PREFIX + "wait_time_ms_between_retry"; public static final String LOCK_ACQUIRE_RETRY_WAIT_TIME_IN_MILLIS_PROP = LOCK_PREFIX + "wait_time_ms_between_retry";
public static final String DEFAULT_LOCK_ACQUIRE_RETRY_WAIT_TIME_IN_MILLIS = String.valueOf(5000L); public static final String DEFAULT_LOCK_ACQUIRE_RETRY_WAIT_TIME_IN_MILLIS = String.valueOf(5000L);
public static final String LOCK_ACQUIRE_RETRY_MAX_WAIT_TIME_IN_MILLIS_PROP = LOCK_PREFIX + "max_wait_time_ms_between_retry"; public static final String LOCK_ACQUIRE_RETRY_MAX_WAIT_TIME_IN_MILLIS_PROP = LOCK_PREFIX + "max_wait_time_ms_between_retry";

18
hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/job/TestHoodieTestSuiteJob.java
View File

@@ -171,15 +171,15 @@ public class TestHoodieTestSuiteJob extends UtilitiesTestBase {
props.setProperty(DataSourceWriteOptions.HIVE_PARTITION_FIELDS_OPT_KEY(), "datestr"); props.setProperty(DataSourceWriteOptions.HIVE_PARTITION_FIELDS_OPT_KEY(), "datestr");
props.setProperty(DataSourceWriteOptions.KEYGENERATOR_CLASS_OPT_KEY(), TimestampBasedKeyGenerator.class.getName()); props.setProperty(DataSourceWriteOptions.KEYGENERATOR_CLASS_OPT_KEY(), TimestampBasedKeyGenerator.class.getName());
props.setProperty("hoodie.writer.lock.provider", "org.apache.hudi.client.transaction.lock.ZookeeperBasedLockProvider"); props.setProperty("hoodie.write.lock.provider", "org.apache.hudi.client.transaction.lock.ZookeeperBasedLockProvider");
props.setProperty("hoodie.writer.lock.hivemetastore.database", "testdb1"); props.setProperty("hoodie.write.lock.hivemetastore.database", "testdb1");
props.setProperty("hoodie.writer.lock.hivemetastore.table", "table1"); props.setProperty("hoodie.write.lock.hivemetastore.table", "table1");
props.setProperty("hoodie.writer.lock.zookeeper.url", "127.0.0.1"); props.setProperty("hoodie.write.lock.zookeeper.url", "127.0.0.1");
props.setProperty("hoodie.writer.lock.zookeeper.port", "2828"); props.setProperty("hoodie.write.lock.zookeeper.port", "2828");
props.setProperty("hoodie.writer.lock.wait_time_ms", "1200000"); props.setProperty("hoodie.write.lock.wait_time_ms", "1200000");
props.setProperty("hoodie.writer.lock.num_retries", "10"); props.setProperty("hoodie.write.lock.num_retries", "10");
props.setProperty("hoodie.writer.lock.zookeeper.lock_key", "test_table"); props.setProperty("hoodie.write.lock.zookeeper.lock_key", "test_table");
props.setProperty("hoodie.writer.lock.zookeeper.zk_base_path", "/test"); props.setProperty("hoodie.write.lock.zookeeper.zk_base_path", "/test");
return props; return props;
} }

34
hudi-utilities/src/test/java/org/apache/hudi/utilities/functional/TestHoodieDeltaStreamer.java
View File

@@ -290,15 +290,15 @@ public class TestHoodieDeltaStreamer extends UtilitiesTestBase {
props.setProperty("include", "base.properties"); props.setProperty("include", "base.properties");
props.setProperty("hoodie.write.concurrency.mode", "optimistic_concurrency_control"); props.setProperty("hoodie.write.concurrency.mode", "optimistic_concurrency_control");
props.setProperty("hoodie.failed.writes.cleaner.policy", "LAZY"); props.setProperty("hoodie.failed.writes.cleaner.policy", "LAZY");
props.setProperty("hoodie.writer.lock.provider", "org.apache.hudi.client.transaction.lock.ZookeeperBasedLockProvider"); props.setProperty("hoodie.write.lock.provider", "org.apache.hudi.client.transaction.lock.ZookeeperBasedLockProvider");
props.setProperty("hoodie.writer.lock.hivemetastore.database", "testdb1"); props.setProperty("hoodie.write.lock.hivemetastore.database", "testdb1");
props.setProperty("hoodie.writer.lock.hivemetastore.table", "table1"); props.setProperty("hoodie.write.lock.hivemetastore.table", "table1");
props.setProperty("hoodie.writer.lock.zookeeper.url", "127.0.0.1"); props.setProperty("hoodie.write.lock.zookeeper.url", "127.0.0.1");
props.setProperty("hoodie.writer.lock.zookeeper.port", "2828"); props.setProperty("hoodie.write.lock.zookeeper.port", "2828");
props.setProperty("hoodie.writer.lock.wait_time_ms", "1200000"); props.setProperty("hoodie.write.lock.wait_time_ms", "1200000");
props.setProperty("hoodie.writer.lock.num_retries", "10"); props.setProperty("hoodie.write.lock.num_retries", "10");
props.setProperty("hoodie.writer.lock.zookeeper.lock_key", "test_table"); props.setProperty("hoodie.write.lock.zookeeper.lock_key", "test_table");
props.setProperty("hoodie.writer.lock.zookeeper.zk_base_path", "/test"); props.setProperty("hoodie.write.lock.zookeeper.zk_base_path", "/test");
UtilitiesTestBase.Helpers.savePropsToDFS(props, dfs, dfsBasePath + "/" + propsFileName); UtilitiesTestBase.Helpers.savePropsToDFS(props, dfs, dfsBasePath + "/" + propsFileName);
return props; return props;
@@ -743,8 +743,8 @@ public class TestHoodieDeltaStreamer extends UtilitiesTestBase {
String tableBasePath = dfsBasePath + "/" + tempDir; String tableBasePath = dfsBasePath + "/" + tempDir;
// enable carrying forward latest checkpoint // enable carrying forward latest checkpoint
TypedProperties props = prepareMultiWriterProps(PROPS_FILENAME_TEST_MULTI_WRITER); TypedProperties props = prepareMultiWriterProps(PROPS_FILENAME_TEST_MULTI_WRITER);
props.setProperty("hoodie.writer.lock.provider", "org.apache.hudi.client.transaction.FileSystemBasedLockProviderTestClass"); props.setProperty("hoodie.write.lock.provider", "org.apache.hudi.client.transaction.FileSystemBasedLockProviderTestClass");
props.setProperty("hoodie.writer.lock.filesystem.path", tableBasePath); props.setProperty("hoodie.write.lock.filesystem.path", tableBasePath);
UtilitiesTestBase.Helpers.savePropsToDFS(props, dfs, dfsBasePath + "/" + PROPS_FILENAME_TEST_MULTI_WRITER); UtilitiesTestBase.Helpers.savePropsToDFS(props, dfs, dfsBasePath + "/" + PROPS_FILENAME_TEST_MULTI_WRITER);
// Keep it higher than batch-size to test continuous mode // Keep it higher than batch-size to test continuous mode
int totalRecords = 3000; int totalRecords = 3000;
@@ -793,8 +793,8 @@ public class TestHoodieDeltaStreamer extends UtilitiesTestBase {
// create new ingestion & backfill job config to generate only INSERTS to avoid conflict // create new ingestion & backfill job config to generate only INSERTS to avoid conflict
props = prepareMultiWriterProps(PROPS_FILENAME_TEST_MULTI_WRITER); props = prepareMultiWriterProps(PROPS_FILENAME_TEST_MULTI_WRITER);
props.setProperty("hoodie.writer.lock.provider", "org.apache.hudi.client.transaction.FileSystemBasedLockProviderTestClass"); props.setProperty("hoodie.write.lock.provider", "org.apache.hudi.client.transaction.FileSystemBasedLockProviderTestClass");
props.setProperty("hoodie.writer.lock.filesystem.path", tableBasePath); props.setProperty("hoodie.write.lock.filesystem.path", tableBasePath);
props.setProperty("hoodie.test.source.generate.inserts", "true"); props.setProperty("hoodie.test.source.generate.inserts", "true");
UtilitiesTestBase.Helpers.savePropsToDFS(props, dfs, dfsBasePath + "/" + PROPS_FILENAME_TEST_MULTI_WRITER); UtilitiesTestBase.Helpers.savePropsToDFS(props, dfs, dfsBasePath + "/" + PROPS_FILENAME_TEST_MULTI_WRITER);
cfgBackfillJob = TestHelpers.makeConfig(tableBasePath, WriteOperationType.INSERT, cfgBackfillJob = TestHelpers.makeConfig(tableBasePath, WriteOperationType.INSERT,
@@ -830,8 +830,8 @@ public class TestHoodieDeltaStreamer extends UtilitiesTestBase {
String tableBasePath = dfsBasePath + "/" + tempDir; String tableBasePath = dfsBasePath + "/" + tempDir;
// enable carrying forward latest checkpoint // enable carrying forward latest checkpoint
TypedProperties props = prepareMultiWriterProps(PROPS_FILENAME_TEST_MULTI_WRITER); TypedProperties props = prepareMultiWriterProps(PROPS_FILENAME_TEST_MULTI_WRITER);
props.setProperty("hoodie.writer.lock.provider", "org.apache.hudi.client.transaction.FileSystemBasedLockProviderTestClass"); props.setProperty("hoodie.write.lock.provider", "org.apache.hudi.client.transaction.FileSystemBasedLockProviderTestClass");
props.setProperty("hoodie.writer.lock.filesystem.path", tableBasePath); props.setProperty("hoodie.write.lock.filesystem.path", tableBasePath);
UtilitiesTestBase.Helpers.savePropsToDFS(props, dfs, dfsBasePath + "/" + PROPS_FILENAME_TEST_MULTI_WRITER); UtilitiesTestBase.Helpers.savePropsToDFS(props, dfs, dfsBasePath + "/" + PROPS_FILENAME_TEST_MULTI_WRITER);
// Keep it higher than batch-size to test continuous mode // Keep it higher than batch-size to test continuous mode
int totalRecords = 3000; int totalRecords = 3000;
@@ -874,8 +874,8 @@ public class TestHoodieDeltaStreamer extends UtilitiesTestBase {
// run the backfill job, enable overriding checkpoint from the latest commit // run the backfill job, enable overriding checkpoint from the latest commit
props = prepareMultiWriterProps(PROPS_FILENAME_TEST_MULTI_WRITER); props = prepareMultiWriterProps(PROPS_FILENAME_TEST_MULTI_WRITER);
props.setProperty("hoodie.writer.lock.provider", "org.apache.hudi.client.transaction.FileSystemBasedLockProviderTestClass"); props.setProperty("hoodie.write.lock.provider", "org.apache.hudi.client.transaction.FileSystemBasedLockProviderTestClass");
props.setProperty("hoodie.writer.lock.filesystem.path", tableBasePath); props.setProperty("hoodie.write.lock.filesystem.path", tableBasePath);
props.setProperty("hoodie.write.meta.key.prefixes", CHECKPOINT_KEY); props.setProperty("hoodie.write.meta.key.prefixes", CHECKPOINT_KEY);
UtilitiesTestBase.Helpers.savePropsToDFS(props, dfs, dfsBasePath + "/" + PROPS_FILENAME_TEST_MULTI_WRITER); UtilitiesTestBase.Helpers.savePropsToDFS(props, dfs, dfsBasePath + "/" + PROPS_FILENAME_TEST_MULTI_WRITER);