lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-1621] Gets the parallelism from context when init StreamWriteOperatorCoordinator (#2579)

Browse Source
This commit is contained in:
lamber-ken
2021-02-17 20:04:38 +08:00
committed by GitHub
parent 37972071ff
commit c4bbcb7f0e
5 changed files with 10 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
hudi-flink/src/main/java/org/apache/hudi/operator/StreamWriteOperatorCoordinator.java
View File

@@ -398,20 +398,20 @@ public class StreamWriteOperatorCoordinator
public static class Provider implements OperatorCoordinator.Provider {
private final OperatorID operatorId;
private final Configuration conf;
private final int numTasks;
public Provider(OperatorID operatorId, Configuration conf, int numTasks) {
public Provider(OperatorID operatorId, Configuration conf) {
this.operatorId = operatorId;
this.conf = conf;
this.numTasks = numTasks;
}
@Override
public OperatorID getOperatorId() {
return this.operatorId;
}
@Override
public OperatorCoordinator create(Context context) {
return new StreamWriteOperatorCoordinator(this.conf, this.numTasks);
return new StreamWriteOperatorCoordinator(this.conf, context.currentParallelism());
}
}
}

7
hudi-flink/src/main/java/org/apache/hudi/operator/StreamWriteOperatorFactory.java
View File

@@ -39,15 +39,12 @@ public class StreamWriteOperatorFactory<I>
private final StreamWriteOperator<I> operator;
private final Configuration conf;
private final int numTasks;
public StreamWriteOperatorFactory(
Configuration conf,
int numTasks) {
Configuration conf) {
super(new StreamWriteOperator<>(conf));
this.operator = (StreamWriteOperator<I>) getOperator();
this.conf = conf;
this.numTasks = numTasks;
}
@Override
@@ -65,7 +62,7 @@ public class StreamWriteOperatorFactory<I>
@Override
public OperatorCoordinator.Provider getCoordinatorProvider(String s, OperatorID operatorID) {
return new StreamWriteOperatorCoordinator.Provider(operatorID, this.conf, this.numTasks);
return new StreamWriteOperatorCoordinator.Provider(operatorID, this.conf);
}
@Override

2
hudi-flink/src/main/java/org/apache/hudi/streamer/HoodieFlinkStreamerV2.java
View File

@@ -74,7 +74,7 @@ public class HoodieFlinkStreamerV2 {
Configuration conf = FlinkOptions.fromStreamerConfig(cfg);
int numWriteTask = conf.getInteger(FlinkOptions.WRITE_TASK_PARALLELISM);
StreamWriteOperatorFactory<HoodieRecord> operatorFactory =
new StreamWriteOperatorFactory<>(conf, numWriteTask);
new StreamWriteOperatorFactory<>(conf);
DataStream<Object> dataStream = env.addSource(new FlinkKafkaConsumer<>(
cfg.kafkaTopic,