lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

Reducing list status calls from listing logfile versions, some associated refactoring

Browse Source
This commit is contained in:
Nishith Agarwal
2018-01-23 15:10:43 -08:00
committed by vinoth chandar
parent 937ae322ba
commit e10100fe32
6 changed files with 20 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
hoodie-common/src/main/java/com/uber/hoodie/common/model/HoodieLogFile.java
View File

@@ -36,6 +36,7 @@ import org.apache.hadoop.fs.Path;
public class HoodieLogFile implements Serializable {
public static final String DELTA_EXTENSION = ".log";
public static final Integer LOGFILE_BASE_VERSION = 1;
private final Path path;
private Optional<FileStatus> fileStatus;

4
hoodie-common/src/main/java/com/uber/hoodie/common/util/FSUtils.java
View File

@@ -284,7 +284,7 @@ public class FSUtils {
Optional<Integer> currentVersion =
getLatestLogVersion(fs, partitionPath, fileId, logFileExtension, baseCommitTime);
// handle potential overflow
return (currentVersion.isPresent()) ? currentVersion.get() : 1;
return (currentVersion.isPresent()) ? currentVersion.get() : HoodieLogFile.LOGFILE_BASE_VERSION;
}
/**
@@ -295,7 +295,7 @@ public class FSUtils {
Optional<Integer> currentVersion =
getLatestLogVersion(fs, partitionPath, fileId, logFileExtension, baseCommitTime);
// handle potential overflow
return (currentVersion.isPresent()) ? currentVersion.get() + 1 : 1;
return (currentVersion.isPresent()) ? currentVersion.get() + 1 : HoodieLogFile.LOGFILE_BASE_VERSION;
}
public static int getDefaultBufferSize(final FileSystem fs) {