lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-3548] Fix if user specify key "hoodie.datasource.clustering.async.enable" directly, async clustering not work (#4905)

Browse Source 
Co-authored-by: Rex An <bonean131@gmail.com>
This commit is contained in:
RexAn
2022-03-04 08:14:07 +08:00
committed by GitHub
parent a4ba0fff07
commit be9a264885
3 changed files with 27 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

26
hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/TestDataSourceUtils.java
View File

@@ -25,6 +25,8 @@ import org.apache.hudi.common.model.HoodieRecord;
import org.apache.hudi.common.model.HoodieRecordPayload;
import org.apache.hudi.common.model.WriteOperationType;
import org.apache.hudi.common.util.Option;
import org.apache.hudi.common.util.collection.ImmutablePair;
import org.apache.hudi.config.HoodieClusteringConfig;
import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.exception.HoodieException;
import org.apache.hudi.execution.bulkinsert.RDDCustomColumnsSortPartitioner;
@@ -222,6 +224,30 @@ public class TestDataSourceUtils {
assertThat(partitioner.isPresent(), is(true));
}
@Test
public void testCreateHoodieConfigWithAsyncClustering() {
ArrayList<ImmutablePair<String, Boolean>> asyncClusteringKeyValues = new ArrayList<>(4);
asyncClusteringKeyValues.add(new ImmutablePair(DataSourceWriteOptions.ASYNC_CLUSTERING_ENABLE().key(), true));
asyncClusteringKeyValues.add(new ImmutablePair(HoodieClusteringConfig.ASYNC_CLUSTERING_ENABLE.key(), true));
asyncClusteringKeyValues.add(new ImmutablePair("hoodie.datasource.clustering.async.enable", true));
asyncClusteringKeyValues.add(new ImmutablePair("hoodie.clustering.async.enabled", true));
asyncClusteringKeyValues.stream().forEach(pair -> {
HashMap<String, String> params = new HashMap<>(3);
params.put(DataSourceWriteOptions.TABLE_TYPE().key(), DataSourceWriteOptions.TABLE_TYPE().defaultValue());
params.put(DataSourceWriteOptions.PAYLOAD_CLASS_NAME().key(),
DataSourceWriteOptions.PAYLOAD_CLASS_NAME().defaultValue());
params.put(pair.left, pair.right.toString());
HoodieWriteConfig hoodieConfig = DataSourceUtils
.createHoodieConfig(avroSchemaString, config.getBasePath(), "test", params);
assertEquals(pair.right, hoodieConfig.isAsyncClusteringEnabled());
TypedProperties prop = new TypedProperties();
prop.putAll(params);
assertEquals(pair.right, HoodieClusteringConfig.from(prop).isAsyncClusteringEnabled());
});
}
@ParameterizedTest
@ValueSource(booleans = {true, false})
public void testBuildHiveSyncConfig(boolean useSyncMode) {