lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-2305] Add MARKERS.type and fix marker-based rollback (#3472)

Browse Source 
- Rollback infers the directory structure and does rollback based on the strategy used while markers were written. "write markers type" in write config is used to determine marker strategy only for new writes.
This commit is contained in:
Y Ethan Guo
2021-08-14 05:18:49 -07:00
committed by GitHub
parent b7da6cb33d
commit 9056c68744
17 changed files with 403 additions and 143 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
hudi-client/hudi-java-client/src/main/java/org/apache/hudi/table/action/rollback/JavaMarkerBasedRollbackStrategy.java
View File

@@ -30,10 +30,9 @@ import org.apache.hudi.common.util.collection.Pair;
import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.exception.HoodieRollbackException;
import org.apache.hudi.table.HoodieTable;
import org.apache.hudi.table.marker.MarkerBasedRollbackUtils;
import org.apache.hudi.table.marker.WriteMarkers;
import org.apache.hudi.table.marker.WriteMarkersFactory;
import java.util.ArrayList;
import java.util.List;
import java.util.stream.Collectors;
@@ -49,9 +48,9 @@ public class JavaMarkerBasedRollbackStrategy<T extends HoodieRecordPayload> exte
@Override
public List<HoodieRollbackStat> execute(HoodieInstant instantToRollback) {
try {
WriteMarkers writeMarkers =
WriteMarkersFactory.get(config.getMarkersType(), table, instantToRollback.getTimestamp());
List<HoodieRollbackStat> rollbackStats = context.map(new ArrayList<>(writeMarkers.allMarkerFilePaths()), markerFilePath -> {
List<String> markerPaths = MarkerBasedRollbackUtils.getAllMarkerPaths(
table, context, instantToRollback.getTimestamp(), config.getRollbackParallelism());
List<HoodieRollbackStat> rollbackStats = context.map(markerPaths, markerFilePath -> {
String typeStr = markerFilePath.substring(markerFilePath.lastIndexOf(".") + 1);
IOType type = IOType.valueOf(typeStr);
switch (type) {