1
0

[HUDI-4018][HUDI-4027] Adding integ test yamls for immutable use-cases. Added delete partition support to integ tests (#5501)

- Added pure immutable test yamls to integ test framework. Added SparkBulkInsertNode as part of it.
- Added delete_partition support to integ test framework using spark-datasource.
- Added a single yaml to test all non core write operations (insert overwrite, insert overwrite table and delete partitions)
- Added tests for 4 concurrent spark datasource writers (multi-writer tests).
- Fixed readme w/ sample commands for multi-writer.
This commit is contained in:
Sivabalan Narayanan
2022-05-12 21:01:55 -04:00
committed by GitHub
parent ecd47e7aae
commit 0cec955fa2
26 changed files with 970 additions and 46 deletions

View File

@@ -212,7 +212,6 @@ object HoodieSparkSqlWriter {
(writeStatuses, client)
}
case WriteOperationType.DELETE_PARTITION => {
val genericRecords = registerKryoClassesAndGetGenericRecords(tblName, sparkContext, df, reconcileSchema)
if (!tableExists) {
throw new HoodieException(s"hoodie table at $basePath does not exist")
}
@@ -222,6 +221,7 @@ object HoodieSparkSqlWriter {
val partitionColsToDelete = parameters(DataSourceWriteOptions.PARTITIONS_TO_DELETE.key()).split(",")
java.util.Arrays.asList(partitionColsToDelete: _*)
} else {
val genericRecords = registerKryoClassesAndGetGenericRecords(tblName, sparkContext, df, reconcileSchema)
genericRecords.map(gr => keyGenerator.getKey(gr).getPartitionPath).toJavaRDD().distinct().collect()
}
// Create a HoodieWriteClient & issue the delete.