lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-400] Check upgrade from old plan to new plan for compaction (#1422)

Browse Source 
* Fix NPE when DataFile is null
* Check from old plan upgrade to new plan
This commit is contained in:
Zhiyuan Zhao
2020-03-20 15:13:17 +08:00
committed by GitHub
parent a752b7b18c
commit 14e0c95206
2 changed files with 3 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
hudi-common/src/main/java/org/apache/hudi/common/versioning/compaction/CompactionV2MigrationHandler.java
View File

@@ -55,7 +55,7 @@ public class CompactionV2MigrationHandler extends AbstractMigratorBase<HoodieCom
v2CompactionOperationList = input.getOperations().stream().map(inp -> v2CompactionOperationList = input.getOperations().stream().map(inp ->
HoodieCompactionOperation.newBuilder().setBaseInstantTime(inp.getBaseInstantTime()) HoodieCompactionOperation.newBuilder().setBaseInstantTime(inp.getBaseInstantTime())
.setFileId(inp.getFileId()).setPartitionPath(inp.getPartitionPath()).setMetrics(inp.getMetrics()) .setFileId(inp.getFileId()).setPartitionPath(inp.getPartitionPath()).setMetrics(inp.getMetrics())
.setDataFilePath(new Path(inp.getDataFilePath()).getName()).setDeltaFilePaths( .setDataFilePath(inp.getDataFilePath() == null ? null : new Path(inp.getDataFilePath()).getName()).setDeltaFilePaths(
inp.getDeltaFilePaths().stream().map(s -> new Path(s).getName()).collect(Collectors.toList())) inp.getDeltaFilePaths().stream().map(s -> new Path(s).getName()).collect(Collectors.toList()))
.build()).collect(Collectors.toList()); .build()).collect(Collectors.toList());
} }

2
hudi-common/src/test/java/org/apache/hudi/common/util/TestCompactionUtils.java
View File

@@ -84,6 +84,8 @@ public class TestCompactionUtils extends HoodieCommonTestHarness {
HoodieCompactionPlan newPlan = migrator.upgradeToLatest(plan, plan.getVersion()); HoodieCompactionPlan newPlan = migrator.upgradeToLatest(plan, plan.getVersion());
Assert.assertEquals(LATEST_COMPACTION_METADATA_VERSION, newPlan.getVersion()); Assert.assertEquals(LATEST_COMPACTION_METADATA_VERSION, newPlan.getVersion());
testFileSlicesCompactionPlanEquality(inputAndPlan.getKey(), newPlan); testFileSlicesCompactionPlanEquality(inputAndPlan.getKey(), newPlan);
HoodieCompactionPlan latestPlan = migrator.migrateToVersion(oldPlan, oldPlan.getVersion(), newPlan.getVersion());
testFileSlicesCompactionPlanEquality(inputAndPlan.getKey(), latestPlan);
} }
@Test @Test