lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

Fix empty content clean plan

Browse Source
This commit is contained in:
lamber-ken
2019-12-29 07:27:59 +08:00
committed by Balaji Varadarajan
parent 2f254163d4
commit 8440482977
2 changed files with 6 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
hudi-client/src/main/java/org/apache/hudi/io/HoodieCommitArchiveLog.java
View File

@@ -249,7 +249,6 @@ public class HoodieCommitArchiveLog {
LOG.info("Wrapper schema " + wrapperSchema.toString()); LOG.info("Wrapper schema " + wrapperSchema.toString());
List<IndexedRecord> records = new ArrayList<>(); List<IndexedRecord> records = new ArrayList<>();
for (HoodieInstant hoodieInstant : instants) { for (HoodieInstant hoodieInstant : instants) {
try { try {
records.add(convertToAvroRecord(commitTimeline, hoodieInstant)); records.add(convertToAvroRecord(commitTimeline, hoodieInstant));
if (records.size() >= this.config.getCommitArchivalBatchSize()) { if (records.size() >= this.config.getCommitArchivalBatchSize()) {
@@ -291,7 +290,7 @@ public class HoodieCommitArchiveLog {
case HoodieTimeline.CLEAN_ACTION: { case HoodieTimeline.CLEAN_ACTION: {
if (hoodieInstant.isCompleted()) { if (hoodieInstant.isCompleted()) {
archivedMetaWrapper.setHoodieCleanMetadata(CleanerUtils.getCleanerMetadata(metaClient, hoodieInstant)); archivedMetaWrapper.setHoodieCleanMetadata(CleanerUtils.getCleanerMetadata(metaClient, hoodieInstant));
} else if (HoodieInstant.State.REQUESTED != hoodieInstant.getState()) { } else {
archivedMetaWrapper.setHoodieCleanerPlan(CleanerUtils.getCleanerPlan(metaClient, hoodieInstant)); archivedMetaWrapper.setHoodieCleanerPlan(CleanerUtils.getCleanerPlan(metaClient, hoodieInstant));
} }
archivedMetaWrapper.setActionType(ActionType.clean.name()); archivedMetaWrapper.setActionType(ActionType.clean.name());
@@ -317,11 +316,9 @@ public class HoodieCommitArchiveLog {
break; break;
} }
case HoodieTimeline.DELTA_COMMIT_ACTION: { case HoodieTimeline.DELTA_COMMIT_ACTION: {
if (HoodieInstant.State.REQUESTED != hoodieInstant.getState()) { HoodieCommitMetadata commitMetadata = HoodieCommitMetadata
HoodieCommitMetadata commitMetadata = HoodieCommitMetadata .fromBytes(commitTimeline.getInstantDetails(hoodieInstant).get(), HoodieCommitMetadata.class);
.fromBytes(commitTimeline.getInstantDetails(hoodieInstant).get(), HoodieCommitMetadata.class); archivedMetaWrapper.setHoodieCommitMetadata(commitMetadataConverter(commitMetadata));
archivedMetaWrapper.setHoodieCommitMetadata(commitMetadataConverter(commitMetadata));
}
archivedMetaWrapper.setActionType(ActionType.commit.name()); archivedMetaWrapper.setActionType(ActionType.commit.name());
break; break;
} }

4
hudi-common/src/main/java/org/apache/hudi/common/table/timeline/HoodieActiveTimeline.java
View File

@@ -472,8 +472,8 @@ public class HoodieActiveTimeline extends HoodieDefaultTimeline {
Preconditions.checkArgument(instant.getState().equals(State.REQUESTED)); Preconditions.checkArgument(instant.getState().equals(State.REQUESTED));
// Write workload to auxiliary folder // Write workload to auxiliary folder
createFileInAuxiliaryFolder(instant, content); createFileInAuxiliaryFolder(instant, content);
// Plan is only stored in auxiliary folder // Plan is only stored in auxiliary folder, here stored in meta path
createFileInMetaPath(instant.getFileName(), Option.empty(), false); createFileInMetaPath(instant.getFileName(), content, false);
} }
private void createFileInMetaPath(String filename, Option<byte[]> content, boolean allowOverwrite) { private void createFileInMetaPath(String filename, Option<byte[]> content, boolean allowOverwrite) {