lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-716] Exception: Not an Avro data file when running HoodieCleanClient.runClean (#1432)

Browse Source
This commit is contained in:
lamber-ken
2020-03-30 13:19:17 -05:00
committed by GitHub
parent 9f51b99174
commit dbc9acd23a
5 changed files with 78 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
hudi-common/src/main/java/org/apache/hudi/common/fs/FSUtils.java
View File

@@ -487,6 +487,15 @@ public class FSUtils {
});
}
public static void deleteInstantFile(FileSystem fs, String metaPath, HoodieInstant instant) {
try {
LOG.warn("try to delete instant file: " + instant);
fs.delete(new Path(metaPath, instant.getFileName()), false);
} catch (IOException e) {
throw new HoodieIOException("Could not delete instant file" + instant.getFileName(), e);
}
}
public static void deleteOlderRestoreMetaFiles(FileSystem fs, String metaPath, Stream<HoodieInstant> instants) {
// TODO - this should be archived when archival is made general for all meta-data
// skip MIN_ROLLBACK_TO_KEEP and delete rest

24
hudi-common/src/test/java/org/apache/hudi/common/model/HoodieTestUtils.java
View File

@@ -179,6 +179,30 @@ public class HoodieTestUtils {
}
}
public static void createCorruptedPendingCleanFiles(HoodieTableMetaClient metaClient, String commitTime) {
Arrays.asList(HoodieTimeline.makeRequestedCleanerFileName(commitTime),
HoodieTimeline.makeInflightCleanerFileName(commitTime)).forEach(f -> {
FSDataOutputStream os = null;
try {
Path commitFile = new Path(
metaClient.getBasePath() + "/" + HoodieTableMetaClient.METAFOLDER_NAME + "/" + f);
os = metaClient.getFs().create(commitFile, true);
// Write empty clean metadata
os.write(new byte[0]);
} catch (IOException ioe) {
throw new HoodieIOException(ioe.getMessage(), ioe);
} finally {
if (null != os) {
try {
os.close();
} catch (IOException e) {
throw new HoodieIOException(e.getMessage(), e);
}
}
}
});
}
public static String createNewDataFile(String basePath, String partitionPath, String instantTime)
throws IOException {
String fileID = UUID.randomUUID().toString();