[HUDI-3647] HoodieMetadataTableValidator: check MDT was initialized at first (#5152)
Co-authored-by: yuezhang <yuezhang@freewheel.tv>
This commit is contained in:
@@ -357,6 +357,9 @@ public class HoodieMetadataTableValidator implements Serializable {
|
||||
String basePath = metaClient.getBasePath();
|
||||
Set<String> baseFilesForCleaning = Collections.emptySet();
|
||||
|
||||
// check metadata table is available to read.
|
||||
checkMetadataTableIsAvailable();
|
||||
|
||||
if (cfg.skipDataFilesForCleaning) {
|
||||
HoodieTimeline inflightCleaningTimeline = metaClient.getActiveTimeline().getCleanerTimeline().filterInflights();
|
||||
|
||||
@@ -415,6 +418,25 @@ public class HoodieMetadataTableValidator implements Serializable {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Check metadata is initialized and available to ready.
|
||||
* If not we will log.warn and skip current validation.
|
||||
*/
|
||||
private void checkMetadataTableIsAvailable() {
|
||||
try {
|
||||
HoodieTableMetaClient mdtMetaClient = HoodieTableMetaClient.builder()
|
||||
.setConf(jsc.hadoopConfiguration()).setBasePath(new Path(cfg.basePath, HoodieTableMetaClient.METADATA_TABLE_FOLDER_PATH).toString())
|
||||
.setLoadActiveTimelineOnLoad(true)
|
||||
.build();
|
||||
int finishedInstants = mdtMetaClient.getActiveTimeline().filterCompletedInstants().countInstants();
|
||||
if (finishedInstants == 0) {
|
||||
throw new HoodieValidationException("There is no completed instant for metadata table.");
|
||||
}
|
||||
} catch (Exception ex) {
|
||||
LOG.warn("Metadata table is not available to ready for now, ", ex);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Compare the listing partitions result between metadata table and fileSystem.
|
||||
*/
|
||||
|
||||
Reference in New Issue
Block a user