lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[MINOR] Removing invalid code to close parquet reader iterator (#5182)

Browse Source
This commit is contained in:
Sivabalan Narayanan
2022-04-15 14:50:07 -04:00
committed by GitHub
parent 57612c5c32
commit e8ab915aff
1 changed files with 1 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieBaseRelation.scala
View File

@@ -422,13 +422,7 @@ object HoodieBaseRelation {
partitionedFile => { partitionedFile => {
val extension = FSUtils.getFileExtension(partitionedFile.filePath) val extension = FSUtils.getFileExtension(partitionedFile.filePath)
if (HoodieFileFormat.PARQUET.getFileExtension.equals(extension)) { if (HoodieFileFormat.PARQUET.getFileExtension.equals(extension)) {
val iter = parquetReader.apply(partitionedFile) parquetReader.apply(partitionedFile)
if (iter.isInstanceOf[Closeable]) {
// register a callback to close parquetReader which will be executed on task completion.
// when tasks finished, this method will be called, and release resources.
Option(TaskContext.get()).foreach(_.addTaskCompletionListener[Unit](_ => iter.asInstanceOf[Closeable].close()))
}
iter
} else if (HoodieFileFormat.HFILE.getFileExtension.equals(extension)) { } else if (HoodieFileFormat.HFILE.getFileExtension.equals(extension)) {
hfileReader.apply(partitionedFile) hfileReader.apply(partitionedFile)
} else { } else {