Implement Savepoints and required metadata timeline (#86)
- Introduce avro to save clean metadata with details about the last commit that was retained - Save rollback metadata in the meta timeline - Create savepoint metadata and add API to createSavepoint, deleteSavepoint and rollbackToSavepoint - Savepointed commit should not be rolledback or cleaned or archived - introduce cli commands to show, create and rollback to savepoints - Write unit tests to test savepoints and rollbackToSavepoints
This commit is contained in:
24
hoodie-common/src/main/avro/HoodieCleanMetadata.avsc
Normal file
24
hoodie-common/src/main/avro/HoodieCleanMetadata.avsc
Normal file
@@ -0,0 +1,24 @@
|
||||
{"namespace": "com.uber.hoodie.avro.model",
|
||||
"type": "record",
|
||||
"name": "HoodieCleanMetadata",
|
||||
"fields": [
|
||||
{"name": "startCleanTime", "type": "string"},
|
||||
{"name": "timeTakenInMillis", "type": "long"},
|
||||
{"name": "totalFilesDeleted", "type": "int"},
|
||||
{"name": "earliestCommitToRetain", "type": "string"},
|
||||
{"name": "partitionMetadata", "type": {
|
||||
"type" : "map", "values" : {
|
||||
"type": "record",
|
||||
"name": "HoodieCleanPartitionMetadata",
|
||||
"fields": [
|
||||
{"name": "partitionPath", "type": "string"},
|
||||
{"name": "policy", "type": "string"},
|
||||
{"name": "deletePathPatterns", "type": {"type": "array", "items": "string"}},
|
||||
{"name": "successDeleteFiles", "type": {"type": "array", "items": "string"}},
|
||||
{"name": "failedDeleteFiles", "type": {"type": "array", "items": "string"}}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
22
hoodie-common/src/main/avro/HoodieRollbackMetadata.avsc
Normal file
22
hoodie-common/src/main/avro/HoodieRollbackMetadata.avsc
Normal file
@@ -0,0 +1,22 @@
|
||||
{"namespace": "com.uber.hoodie.avro.model",
|
||||
"type": "record",
|
||||
"name": "HoodieRollbackMetadata",
|
||||
"fields": [
|
||||
{"name": "startRollbackTime", "type": "string"},
|
||||
{"name": "timeTakenInMillis", "type": "long"},
|
||||
{"name": "totalFilesDeleted", "type": "int"},
|
||||
{"name": "commitsRollback", "type": {"type": "array", "items": "string"}},
|
||||
{"name": "partitionMetadata", "type": {
|
||||
"type" : "map", "values" : {
|
||||
"type": "record",
|
||||
"name": "HoodieRollbackPartitionMetadata",
|
||||
"fields": [
|
||||
{"name": "partitionPath", "type": "string"},
|
||||
{"name": "successDeleteFiles", "type": {"type": "array", "items": "string"}},
|
||||
{"name": "failedDeleteFiles", "type": {"type": "array", "items": "string"}}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
9
hoodie-common/src/main/avro/HoodieSavePointMetadata.avsc
Normal file
9
hoodie-common/src/main/avro/HoodieSavePointMetadata.avsc
Normal file
@@ -0,0 +1,9 @@
|
||||
{"namespace": "com.uber.hoodie.avro.model",
|
||||
"type": "record",
|
||||
"name": "HoodieSavepointMetadata",
|
||||
"fields": [
|
||||
{"name": "savepointedBy", "type": "string"},
|
||||
{"name": "savepointedAt", "type": "string"},
|
||||
{"name": "comments", "type": "string"}
|
||||
]
|
||||
}
|
||||
Reference in New Issue
Block a user