80011df995
As of now, delete partitions will ensure all file groups are deleted, but the partition as such is not deleted. So, get all partitions might be returning the deleted partitions as well. but no data will be served since all file groups are deleted. With this patch, we are fixing it. We are letting cleaner take care of deleting the partitions when all file groups pertaining to a partitions are deleted. - Fixed the CleanPlanActionExecutor to return meta info about list of partitions to be deleted. If there are no valid file groups for a partition, clean planner will include the partition to be deleted. - Fixed HoodieCleanPlan avro schema to include the list of partitions to be deleted - CleanActionExecutor is fixed to delete partitions if any (as per clean plan) - Same info is added to HoodieCleanMetadata - Metadata table when applying clean metadata, will check for partitions to be deleted and will update the "all_partitions" record for the deleted partitions. Co-authored-by: sivabalan <n.siva.b@gmail.com>
106 lines
2.8 KiB
JSON
106 lines
2.8 KiB
JSON
/*
|
|
* Licensed to the Apache Software Foundation (ASF) under one
|
|
* or more contributor license agreements. See the NOTICE file
|
|
* distributed with this work for additional information
|
|
* regarding copyright ownership. The ASF licenses this file
|
|
* to you under the Apache License, Version 2.0 (the
|
|
* "License"); you may not use this file except in compliance
|
|
* with the License. You may obtain a copy of the License at
|
|
*
|
|
* http://www.apache.org/licenses/LICENSE-2.0
|
|
*
|
|
* Unless required by applicable law or agreed to in writing, software
|
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
* See the License for the specific language governing permissions and
|
|
* limitations under the License.
|
|
*/
|
|
{
|
|
"namespace": "org.apache.hudi.avro.model",
|
|
"type": "record",
|
|
"name": "HoodieCleanerPlan",
|
|
"fields": [
|
|
{
|
|
"name": "earliestInstantToRetain",
|
|
"type":["null", {
|
|
"type": "record",
|
|
"name": "HoodieActionInstant",
|
|
"fields": [
|
|
{
|
|
"name": "timestamp",
|
|
"type": "string"
|
|
},
|
|
{
|
|
"name": "action",
|
|
"type": "string"
|
|
},
|
|
{
|
|
"name": "state",
|
|
"type": "string"
|
|
}
|
|
]
|
|
}],
|
|
"default" : null
|
|
},
|
|
{
|
|
"name": "policy",
|
|
"type": "string"
|
|
},
|
|
{
|
|
/** This is deprecated and replaced by the field filePathsToBeDeletedPerPartition **/
|
|
"name": "filesToBeDeletedPerPartition",
|
|
"type": [
|
|
"null", {
|
|
"type":"map",
|
|
"values": {
|
|
"type":"array",
|
|
"items":{
|
|
"name":"filePath",
|
|
"type": "string"
|
|
}
|
|
}}],
|
|
"default" : null
|
|
},
|
|
{
|
|
"name":"version",
|
|
"type":["int", "null"],
|
|
"default": 1
|
|
},
|
|
{
|
|
"name": "filePathsToBeDeletedPerPartition",
|
|
"doc": "This field replaces the field filesToBeDeletedPerPartition",
|
|
"type": [
|
|
"null", {
|
|
"type":"map",
|
|
"values": {
|
|
"type":"array",
|
|
"items":{
|
|
"name":"HoodieCleanFileInfo",
|
|
"type": "record",
|
|
"fields":[
|
|
{
|
|
"name":"filePath",
|
|
"type":["null","string"],
|
|
"default": null
|
|
},
|
|
{
|
|
"name":"isBootstrapBaseFile",
|
|
"type":["null","boolean"],
|
|
"default": null
|
|
}
|
|
]
|
|
}
|
|
}}],
|
|
"default" : null
|
|
},
|
|
{
|
|
"name": "partitionsToBeDeleted",
|
|
"doc": "partitions to be deleted",
|
|
"type":["null",
|
|
{ "type":"array", "items":"string"}
|
|
],
|
|
"default": null
|
|
}
|
|
]
|
|
}
|