lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[TEST][DO_NOT_MERGE]fix random failed for ci (#5948)

Browse Source
This commit is contained in:
xiarixiaoyao
2022-06-25 10:15:08 +08:00
committed by GitHub
parent eeafaeacd2
commit e8fbd4daf4
1 changed files with 4 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/BaseFileOnlyRelation.scala
View File

@@ -166,7 +166,9 @@ class BaseFileOnlyRelation(sqlContext: SQLContext,
DataSource.apply( DataSource.apply(
sparkSession = sparkSession, sparkSession = sparkSession,
paths = extraReadPaths, paths = extraReadPaths,
userSpecifiedSchema = userSchema, // Here we should specify the schema to the latest commit schema since
// the table schema evolution.
userSpecifiedSchema = userSchema.orElse(Some(tableStructSchema)),
className = formatClassName, className = formatClassName,
// Since we're reading the table as just collection of files we have to make sure // Since we're reading the table as just collection of files we have to make sure
// we only read the latest version of every Hudi's file-group, which might be compacted, clustered, etc. // we only read the latest version of every Hudi's file-group, which might be compacted, clustered, etc.
@@ -175,8 +177,7 @@ class BaseFileOnlyRelation(sqlContext: SQLContext,
// We rely on [[HoodieROTablePathFilter]], to do proper filtering to assure that // We rely on [[HoodieROTablePathFilter]], to do proper filtering to assure that
options = optParams ++ Map( options = optParams ++ Map(
"mapreduce.input.pathFilter.class" -> classOf[HoodieROTablePathFilter].getName "mapreduce.input.pathFilter.class" -> classOf[HoodieROTablePathFilter].getName
), )
partitionColumns = partitionColumns
) )
.resolveRelation() .resolveRelation()
.asInstanceOf[HadoopFsRelation] .asInstanceOf[HadoopFsRelation]