1
0

[HUDI-3647] HoodieMetadataTableValidator: check MDT was initialized at first (#5152)

Co-authored-by: yuezhang <yuezhang@freewheel.tv>
This commit is contained in:
YueZhang
2022-03-31 05:18:08 +08:00
committed by GitHub
parent 8b796e9686
commit eae8488536

View File

@@ -357,6 +357,9 @@ public class HoodieMetadataTableValidator implements Serializable {
String basePath = metaClient.getBasePath();
Set<String> baseFilesForCleaning = Collections.emptySet();
// check metadata table is available to read.
checkMetadataTableIsAvailable();
if (cfg.skipDataFilesForCleaning) {
HoodieTimeline inflightCleaningTimeline = metaClient.getActiveTimeline().getCleanerTimeline().filterInflights();
@@ -415,6 +418,25 @@ public class HoodieMetadataTableValidator implements Serializable {
}
}
/**
* Check metadata is initialized and available to ready.
* If not we will log.warn and skip current validation.
*/
private void checkMetadataTableIsAvailable() {
try {
HoodieTableMetaClient mdtMetaClient = HoodieTableMetaClient.builder()
.setConf(jsc.hadoopConfiguration()).setBasePath(new Path(cfg.basePath, HoodieTableMetaClient.METADATA_TABLE_FOLDER_PATH).toString())
.setLoadActiveTimelineOnLoad(true)
.build();
int finishedInstants = mdtMetaClient.getActiveTimeline().filterCompletedInstants().countInstants();
if (finishedInstants == 0) {
throw new HoodieValidationException("There is no completed instant for metadata table.");
}
} catch (Exception ex) {
LOG.warn("Metadata table is not available to ready for now, ", ex);
}
}
/**
* Compare the listing partitions result between metadata table and fileSystem.
*/