lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-3435] Do not throw exception when instant to rollback does not exist in metadata table active timeline (#4821)

Browse Source
This commit is contained in:
Danny Chan
2022-03-26 11:42:54 +08:00
committed by GitHub
parent 51034fecf1
commit 0c09a973fb
5 changed files with 108 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
hudi-common/src/main/java/org/apache/hudi/metadata/HoodieTableMetadata.java
View File

@@ -27,6 +27,7 @@ import org.apache.hudi.common.config.SerializableConfiguration;
import org.apache.hudi.common.engine.HoodieEngineContext;
import org.apache.hudi.common.table.HoodieTableMetaClient;
import org.apache.hudi.common.util.Option;
import org.apache.hudi.common.util.collection.Pair;
import org.apache.hudi.exception.HoodieMetadataException;
@@ -36,6 +37,7 @@ import java.util.List;
import java.util.Map;
import static org.apache.hudi.common.util.ValidationUtils.checkArgument;
import static org.apache.hudi.common.util.ValidationUtils.checkState;
/**
* Interface that supports querying various pieces of metadata about a hudi table.
@@ -72,6 +74,17 @@ public interface HoodieTableMetadata extends Serializable, AutoCloseable {
return metadataTableBasePath.substring(0, metadataTableBasePath.lastIndexOf(HoodieTableMetaClient.METADATA_TABLE_FOLDER_PATH) - 1);
}
/**
* Return the base path of the dataset.
*
* @param metadataTableBasePath The base path of the metadata table
*/
static String getDatasetBasePath(String metadataTableBasePath) {
int endPos = metadataTableBasePath.lastIndexOf(Path.SEPARATOR + HoodieTableMetaClient.METADATA_TABLE_FOLDER_PATH);
checkState(endPos != -1, metadataTableBasePath + " should be base path of the metadata table");
return metadataTableBasePath.substring(0, endPos);
}
/**
* Returns {@code True} if the given path contains a metadata table.
*

27
hudi-common/src/main/java/org/apache/hudi/metadata/HoodieTableMetadataUtil.java
View File

@@ -531,14 +531,35 @@ public class HoodieTableMetadataUtil {
}
// Case 2: The instant-to-rollback was never committed to Metadata Table. This can happen if the instant-to-rollback
// was a failed commit (never completed) as only completed instants are synced to Metadata Table.
// But the required Metadata Table instants should not have been archived
// was a failed commit (never completed).
//
// There are two cases for failed commit that we need to take care of:
// 1) The commit was synced to metadata table successfully but the dataset meta file switches state failed
// (from INFLIGHT to COMPLETED), the committed files should be rolled back thus the rollback metadata
// can not be skipped, usually a failover should be triggered and the metadata active timeline expects
// to contain the commit, we could check whether the commit was synced to metadata table
// through HoodieActiveTimeline#containsInstant.
//
// 2) The commit synced to metadata table failed or was never synced to metadata table,
// in this case, the rollback metadata should be skipped.
//
// And in which case,
// metadataTableTimeline.getCommitsTimeline().isBeforeTimelineStarts(syncedInstant.getTimestamp())
// returns true ?
// It is most probably because of compaction rollback, we schedule a compaction plan early in the timeline (say t1)
// then after a long time schedule and execute the plan then try to rollback it.
//
// scheduled execution rollback compaction actions
// ----- t1 ----- t3 ----- t4 ----- dataset timeline
//
// ---------- t2 (archive) ----------- metadata timeline
//
// when at time t4, we commit the compaction rollback,the above check returns true.
HoodieInstant syncedInstant = new HoodieInstant(false, HoodieTimeline.DELTA_COMMIT_ACTION, instantToRollback);
if (metadataTableTimeline.getCommitsTimeline().isBeforeTimelineStarts(syncedInstant.getTimestamp())) {
throw new HoodieMetadataException(String.format("The instant %s required to sync rollback of %s has been archived",
syncedInstant, instantToRollback));
}
shouldSkip = !metadataTableTimeline.containsInstant(syncedInstant);
if (!hasNonZeroRollbackLogFiles && shouldSkip) {
LOG.info(String.format("Skipping syncing of rollbackMetadata at %s, since this instant was never committed to Metadata Table",