lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-2047] Ignore FileNotFoundException in WriteProfiles #getWritePathsOfInstant (#3125)

Browse Source 
Co-authored-by: 喻兆靖 <yuzhaojing@bilibili.com>
This commit is contained in:
yuzhaojing
2021-06-22 14:18:46 +08:00
committed by GitHub
parent 7bd517a82f
commit 5db37c255b
1 changed files with 6 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
hudi-flink/src/main/java/org/apache/hudi/sink/partitioner/profile/WriteProfiles.java
View File

@@ -34,11 +34,13 @@ import org.apache.hadoop.fs.FileSystem;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import java.io.FileNotFoundException;
import java.io.IOException; import java.io.IOException;
import java.util.Collection; import java.util.Collection;
import java.util.HashMap; import java.util.HashMap;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Objects;
import java.util.stream.Collectors; import java.util.stream.Collectors;
/** /**
@@ -119,12 +121,16 @@ public class WriteProfiles {
}).map(path -> { }).map(path -> {
try { try {
return fs.getFileStatus(path); return fs.getFileStatus(path);
} catch (FileNotFoundException fe) {
LOG.warn("File {} was deleted by the cleaner, ignore", path);
return null;
} catch (IOException e) { } catch (IOException e) {
LOG.error("Get write status of path: {} error", path); LOG.error("Get write status of path: {} error", path);
throw new HoodieException(e); throw new HoodieException(e);
} }
}) })
// filter out crushed files // filter out crushed files
.filter(Objects::nonNull)
.filter(fileStatus -> fileStatus.getLen() > 0) .filter(fileStatus -> fileStatus.getLen() > 0)
.collect(Collectors.toList()); .collect(Collectors.toList());
} }