lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-3485] Adding scheduler pool configs for async clustering (#5043)

Browse Source
This commit is contained in:
Sivabalan Narayanan
2022-03-29 18:27:45 -07:00
committed by GitHub
parent 5c1b482a1b
commit 4fed8dd319
10 changed files with 72 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/async/SparkAsyncClusteringService.java
View File

@@ -22,14 +22,15 @@ package org.apache.hudi.async;
import org.apache.hudi.client.BaseClusterer;
import org.apache.hudi.client.BaseHoodieWriteClient;
import org.apache.hudi.client.HoodieSparkClusteringClient;
import org.apache.hudi.common.engine.HoodieEngineContext;
/**
* Async clustering service for Spark datasource.
*/
public class SparkAsyncClusteringService extends AsyncClusteringService {
public SparkAsyncClusteringService(BaseHoodieWriteClient writeClient) {
super(writeClient);
public SparkAsyncClusteringService(HoodieEngineContext engineContext, BaseHoodieWriteClient writeClient) {
super(engineContext, writeClient);
}
@Override

5
hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/common/HoodieSparkEngineContext.java
View File

@@ -36,6 +36,8 @@ import org.apache.hudi.data.HoodieJavaRDD;
import org.apache.hudi.data.HoodieSparkLongAccumulator;
import org.apache.hudi.exception.HoodieException;
import org.apache.log4j.LogManager;
import org.apache.log4j.Logger;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.api.java.function.PairFlatMapFunction;
import org.apache.spark.sql.SQLContext;
@@ -53,6 +55,7 @@ import java.util.stream.Stream;
*/
public class HoodieSparkEngineContext extends HoodieEngineContext {
private static final Logger LOG = LogManager.getLogger(HoodieSparkEngineContext.class);
private final JavaSparkContext javaSparkContext;
private SQLContext sqlContext;
@@ -158,6 +161,8 @@ public class HoodieSparkEngineContext extends HoodieEngineContext {
public void setProperty(EngineProperty key, String value) {
if (key == EngineProperty.COMPACTION_POOL_NAME) {
javaSparkContext.setLocalProperty("spark.scheduler.pool", value);
} else if (key == EngineProperty.CLUSTERING_POOL_NAME) {
javaSparkContext.setLocalProperty("spark.scheduler.pool", value);
} else {
throw new HoodieException("Unknown engine property :" + key);
}