lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-2297] Estimate available memory size for spillable map accurately. (#3455)

Browse Source
This commit is contained in:
Shawy Geng
2021-11-10 14:05:12 +08:00
committed by GitHub
parent bb6a19e7d7
commit fd0f5df26d
3 changed files with 20 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/SparkTaskContextSupplier.java
View File

@@ -78,6 +78,14 @@ public class SparkTaskContextSupplier extends TaskContextSupplier implements Ser
.get(SPARK_EXECUTOR_MEMORY_FRACTION_PROP, DEFAULT_SPARK_EXECUTOR_MEMORY_FRACTION));
}
return Option.empty();
} else if (prop == EngineProperty.TOTAL_CORES_PER_EXECUTOR) {
final String DEFAULT_SPARK_EXECUTOR_CORES = "1";
final String SPARK_EXECUTOR_EXECUTOR_CORES_PROP = "spark.executor.cores";
if (SparkEnv.get() != null) {
return Option.ofNullable(SparkEnv.get().conf()
.get(SPARK_EXECUTOR_EXECUTOR_CORES_PROP, DEFAULT_SPARK_EXECUTOR_CORES));
}
return Option.empty();
}
throw new HoodieException("Unknown engine property :" + prop);
}