1
0

[HUDI-4309] fix spark32 repartition error (#6033)

This commit is contained in:
KnightChess
2022-07-08 09:38:09 +08:00
committed by GitHub
parent e74ad324c3
commit 5673819736
3 changed files with 66 additions and 10 deletions

View File

@@ -250,4 +250,48 @@ class TestTimeTravelTable extends HoodieSparkSqlTestBase {
}
}
}
test("Test Select Record with time travel and Repartition") {
if (HoodieSparkUtils.gteqSpark3_2) {
withTempDir { tmp =>
val tableName = generateTableName
spark.sql(
s"""
|create table $tableName (
| id int,
| name string,
| price double,
| ts long
|) using hudi
| tblproperties (
| type = 'cow',
| primaryKey = 'id',
| preCombineField = 'ts'
| )
| location '${tmp.getCanonicalPath}/$tableName'
""".stripMargin)
spark.sql(s"insert into $tableName values(1, 'a1', 10, 1000)")
val metaClient = HoodieTableMetaClient.builder()
.setBasePath(s"${tmp.getCanonicalPath}/$tableName")
.setConf(spark.sessionState.newHadoopConf())
.build()
val instant = metaClient.getActiveTimeline.getAllCommitsTimeline
.lastInstant().get().getTimestamp
spark.sql(s"insert into $tableName values(1, 'a2', 20, 2000)")
checkAnswer(s"select id, name, price, ts from $tableName distribute by cast(rand() * 2 as int)")(
Seq(1, "a2", 20.0, 2000)
)
// time travel from instant
checkAnswer(
s"select id, name, price, ts from $tableName TIMESTAMP AS OF '$instant' distribute by cast(rand() * 2 as int)")(
Seq(1, "a1", 10.0, 1000)
)
}
}
}
}