1
0

[HUDI-4296] Fix the bug that TestHoodieSparkSqlWriter.testSchemaEvolutionForTableType is flaky (#5973)

This commit is contained in:
xiarixiaoyao
2022-06-25 21:03:19 +08:00
committed by GitHub
parent c86edfc28e
commit 142adf4ccb

View File

@@ -166,7 +166,9 @@ class BaseFileOnlyRelation(sqlContext: SQLContext,
DataSource.apply(
sparkSession = sparkSession,
paths = extraReadPaths,
userSpecifiedSchema = userSchema,
// Here we should specify the schema to the latest commit schema since
// the table schema evolution.
userSpecifiedSchema = userSchema.orElse(Some(tableStructSchema)),
className = formatClassName,
// Since we're reading the table as just collection of files we have to make sure
// we only read the latest version of every Hudi's file-group, which might be compacted, clustered, etc.
@@ -175,8 +177,7 @@ class BaseFileOnlyRelation(sqlContext: SQLContext,
// We rely on [[HoodieROTablePathFilter]], to do proper filtering to assure that
options = optParams ++ Map(
"mapreduce.input.pathFilter.class" -> classOf[HoodieROTablePathFilter].getName
),
partitionColumns = partitionColumns
)
)
.resolveRelation()
.asInstanceOf[HadoopFsRelation]