lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-716] Exception: Not an Avro data file when running HoodieCleanClient.runClean (#1432)

Browse Source
This commit is contained in:
lamber-ken
2020-03-30 13:19:17 -05:00
committed by GitHub
parent 9f51b99174
commit dbc9acd23a
5 changed files with 78 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
hudi-client/src/main/java/org/apache/hudi/client/HoodieCleanClient.java
View File

@@ -85,7 +85,11 @@ public class HoodieCleanClient<T extends HoodieRecordPayload> extends AbstractHo
// If there are inflight(failed) or previously requested clean operation, first perform them
table.getCleanTimeline().filterInflightsAndRequested().getInstants().forEach(hoodieInstant -> {
LOG.info("There were previously unfinished cleaner operations. Finishing Instant=" + hoodieInstant);
runClean(table, hoodieInstant);
try {
runClean(table, hoodieInstant);
} catch (Exception e) {
LOG.warn("Failed to perform previous clean operation, instant: " + hoodieInstant, e);
}
});
Option<HoodieCleanerPlan> cleanerPlanOpt = scheduleClean(startCleanTime);

19
hudi-client/src/test/java/org/apache/hudi/table/TestCleaner.java
View File

@@ -984,6 +984,25 @@ public class TestCleaner extends TestHoodieClientBase {
testPendingCompactions(config, 36, 9, retryFailure);
}
/**
* Test clean previous corrupted cleanFiles.
*/
@Test
public void testCleanPreviousCorruptedCleanFiles() {
HoodieWriteConfig config =
HoodieWriteConfig.newBuilder()
.withPath(basePath).withAssumeDatePartitioning(true)
.withCompactionConfig(HoodieCompactionConfig.newBuilder()
.withCleanerPolicy(HoodieCleaningPolicy.KEEP_LATEST_FILE_VERSIONS).retainFileVersions(1).build())
.build();
HoodieTestUtils.createCorruptedPendingCleanFiles(metaClient, getNextInstant());
metaClient = HoodieTableMetaClient.reload(metaClient);
List<HoodieCleanStat> cleanStats = runCleaner(config);
assertEquals("Must not clean any files", 0, cleanStats.size());
}
/**
* Common test method for validating pending compactions.
*