[HUDI-4018][HUDI-4027] Adding integ test yamls for immutable use-cases. Added delete partition support to integ tests (#5501)
- Added pure immutable test yamls to integ test framework. Added SparkBulkInsertNode as part of it. - Added delete_partition support to integ test framework using spark-datasource. - Added a single yaml to test all non core write operations (insert overwrite, insert overwrite table and delete partitions) - Added tests for 4 concurrent spark datasource writers (multi-writer tests). - Fixed readme w/ sample commands for multi-writer.
This commit is contained in:
committed by
GitHub
parent
ecd47e7aae
commit
0cec955fa2
@@ -212,7 +212,6 @@ object HoodieSparkSqlWriter {
|
||||
(writeStatuses, client)
|
||||
}
|
||||
case WriteOperationType.DELETE_PARTITION => {
|
||||
val genericRecords = registerKryoClassesAndGetGenericRecords(tblName, sparkContext, df, reconcileSchema)
|
||||
if (!tableExists) {
|
||||
throw new HoodieException(s"hoodie table at $basePath does not exist")
|
||||
}
|
||||
@@ -222,6 +221,7 @@ object HoodieSparkSqlWriter {
|
||||
val partitionColsToDelete = parameters(DataSourceWriteOptions.PARTITIONS_TO_DELETE.key()).split(",")
|
||||
java.util.Arrays.asList(partitionColsToDelete: _*)
|
||||
} else {
|
||||
val genericRecords = registerKryoClassesAndGetGenericRecords(tblName, sparkContext, df, reconcileSchema)
|
||||
genericRecords.map(gr => keyGenerator.getKey(gr).getPartitionPath).toJavaRDD().distinct().collect()
|
||||
}
|
||||
// Create a HoodieWriteClient & issue the delete.
|
||||
|
||||
Reference in New Issue
Block a user