From 14e0c95206f6d7c1806555490bcbce8785ffea5a Mon Sep 17 00:00:00 2001 From: Zhiyuan Zhao <49054376+zhaomin1423@users.noreply.github.com> Date: Fri, 20 Mar 2020 15:13:17 +0800 Subject: [PATCH] [HUDI-400] Check upgrade from old plan to new plan for compaction (#1422) * Fix NPE when DataFile is null * Check from old plan upgrade to new plan --- .../versioning/compaction/CompactionV2MigrationHandler.java | 2 +- .../java/org/apache/hudi/common/util/TestCompactionUtils.java | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/hudi-common/src/main/java/org/apache/hudi/common/versioning/compaction/CompactionV2MigrationHandler.java b/hudi-common/src/main/java/org/apache/hudi/common/versioning/compaction/CompactionV2MigrationHandler.java index 26180e44a..0c07b7cf5 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/versioning/compaction/CompactionV2MigrationHandler.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/versioning/compaction/CompactionV2MigrationHandler.java @@ -55,7 +55,7 @@ public class CompactionV2MigrationHandler extends AbstractMigratorBase HoodieCompactionOperation.newBuilder().setBaseInstantTime(inp.getBaseInstantTime()) .setFileId(inp.getFileId()).setPartitionPath(inp.getPartitionPath()).setMetrics(inp.getMetrics()) - .setDataFilePath(new Path(inp.getDataFilePath()).getName()).setDeltaFilePaths( + .setDataFilePath(inp.getDataFilePath() == null ? null : new Path(inp.getDataFilePath()).getName()).setDeltaFilePaths( inp.getDeltaFilePaths().stream().map(s -> new Path(s).getName()).collect(Collectors.toList())) .build()).collect(Collectors.toList()); } diff --git a/hudi-common/src/test/java/org/apache/hudi/common/util/TestCompactionUtils.java b/hudi-common/src/test/java/org/apache/hudi/common/util/TestCompactionUtils.java index 2d3cf2e37..034ac3cad 100644 --- a/hudi-common/src/test/java/org/apache/hudi/common/util/TestCompactionUtils.java +++ b/hudi-common/src/test/java/org/apache/hudi/common/util/TestCompactionUtils.java @@ -84,6 +84,8 @@ public class TestCompactionUtils extends HoodieCommonTestHarness { HoodieCompactionPlan newPlan = migrator.upgradeToLatest(plan, plan.getVersion()); Assert.assertEquals(LATEST_COMPACTION_METADATA_VERSION, newPlan.getVersion()); testFileSlicesCompactionPlanEquality(inputAndPlan.getKey(), newPlan); + HoodieCompactionPlan latestPlan = migrator.migrateToVersion(oldPlan, oldPlan.getVersion(), newPlan.getVersion()); + testFileSlicesCompactionPlanEquality(inputAndPlan.getKey(), latestPlan); } @Test