lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

Merge pull request #6154 from rahil-c/rahil-c/disable-emrSpark-properties

Browse Source 
[HUDI-4434] Disable EmrFS file metadata caching and EMR Spark's data prefetcher feature
This commit is contained in:
wenningd
2022-07-21 11:35:52 -07:00
committed by GitHub
parent 2bf7920bd9 473be87aa5
commit f52b93fd10
1 changed files with 3 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/DefaultSource.scala
View File

@@ -57,6 +57,9 @@ class DefaultSource extends RelationProvider
// Enable "passPartitionByAsOptions" to support "write.partitionBy(...)" // Enable "passPartitionByAsOptions" to support "write.partitionBy(...)"
spark.conf.set("spark.sql.legacy.sources.write.passPartitionByAsOptions", "true") spark.conf.set("spark.sql.legacy.sources.write.passPartitionByAsOptions", "true")
} }
// Revisit EMR Spark and EMRFS incompatibilities, for now disable
spark.conf.set("spark.sql.dataPrefetch.enabled", "false")
spark.sparkContext.hadoopConfiguration.set("fs.s3.metadata.cache.expiration.seconds", "0")
} }
private val log = LogManager.getLogger(classOf[DefaultSource]) private val log = LogManager.getLogger(classOf[DefaultSource])