lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

Writes relative paths to .commit files

Browse Source 
Handle case where path is read in as null from commit file

Merged with updated release
This commit is contained in:
gekath
2017-06-13 13:35:38 -04:00
committed by prazanna
parent db7311f85e
commit 52c507f83e
4 changed files with 9 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
hoodie-client/src/main/java/com/uber/hoodie/io/HoodieAppendHandle.java
View File

@@ -80,8 +80,6 @@ public class HoodieAppendHandle<T extends HoodieRecordPayload> extends HoodieIOH
fileSystemView.getLatestDataFilesForFileId(record.getPartitionPath(), fileId) fileSystemView.getLatestDataFilesForFileId(record.getPartitionPath(), fileId)
.findFirst().get().getFileName(); .findFirst().get().getFileName();
String baseCommitTime = FSUtils.getCommitTime(latestValidFilePath); String baseCommitTime = FSUtils.getCommitTime(latestValidFilePath);
Path path = new Path(record.getPartitionPath(),
FSUtils.makeDataFileName(commitTime, TaskContext.getPartitionId(), fileId));
writeStatus.getStat().setPrevCommit(baseCommitTime); writeStatus.getStat().setPrevCommit(baseCommitTime);
writeStatus.setFileId(fileId); writeStatus.setFileId(fileId);
writeStatus.setPartitionPath(record.getPartitionPath()); writeStatus.setPartitionPath(record.getPartitionPath());
@@ -105,6 +103,8 @@ public class HoodieAppendHandle<T extends HoodieRecordPayload> extends HoodieIOH
+ " on commit " + commitTime + " on HDFS path " + hoodieTable + " on commit " + commitTime + " on HDFS path " + hoodieTable
.getMetaClient().getBasePath() + partitionPath, e); .getMetaClient().getBasePath() + partitionPath, e);
} }
Path path = new Path(record.getPartitionPath(),
FSUtils.makeDataFileName(commitTime, TaskContext.getPartitionId(), fileId));
writeStatus.getStat().setPath(path.toString()); writeStatus.getStat().setPath(path.toString());
} }
// update the new location of the record, so we know where to find it next // update the new location of the record, so we know where to find it next

3
hoodie-client/src/main/java/com/uber/hoodie/io/HoodieInsertHandle.java
View File

@@ -123,13 +123,12 @@ public class HoodieInsertHandle<T extends HoodieRecordPayload> extends HoodieIOH
try { try {
storageWriter.close(); storageWriter.close();
String relativePath = path.toString().replace(new Path(config.getBasePath()) + "/", "");
HoodieWriteStat stat = new HoodieWriteStat(); HoodieWriteStat stat = new HoodieWriteStat();
stat.setNumWrites(recordsWritten); stat.setNumWrites(recordsWritten);
stat.setNumDeletes(recordsDeleted); stat.setNumDeletes(recordsDeleted);
stat.setPrevCommit(HoodieWriteStat.NULL_COMMIT); stat.setPrevCommit(HoodieWriteStat.NULL_COMMIT);
stat.setFileId(status.getFileId()); stat.setFileId(status.getFileId());
String relativePath = path.toString().replace(new Path(config.getBasePath()) + "/", "");
stat.setPath(relativePath); stat.setPath(relativePath);
stat.setTotalWriteBytes(FSUtils.getFileSize(fs, path)); stat.setTotalWriteBytes(FSUtils.getFileSize(fs, path));
stat.setTotalWriteErrors(status.getFailedRecords().size()); stat.setTotalWriteErrors(status.getFailedRecords().size());

4
hoodie-client/src/test/java/com/uber/hoodie/TestHoodieClient.java
View File

@@ -1137,6 +1137,7 @@ public class TestHoodieClient implements Serializable {
List<HoodieCleanStat> hoodieCleanStatsFour = table.clean(jsc); List<HoodieCleanStat> hoodieCleanStatsFour = table.clean(jsc);
assertEquals("Must not clean any files" , 0, getCleanStat(hoodieCleanStatsFour, partitionPaths[0]).getSuccessDeleteFiles().size()); assertEquals("Must not clean any files" , 0, getCleanStat(hoodieCleanStatsFour, partitionPaths[0]).getSuccessDeleteFiles().size());
assertTrue(HoodieTestUtils.doesDataFileExist(basePath, partitionPaths[0], "002", file3P0C2)); assertTrue(HoodieTestUtils.doesDataFileExist(basePath, partitionPaths[0], "002", file3P0C2));
}
@Test @Test
public void testKeepLatestCommits() throws IOException { public void testKeepLatestCommits() throws IOException {
@@ -1335,9 +1336,10 @@ public class TestHoodieClient implements Serializable {
inputStream.close(); inputStream.close();
// Compare values in both to make sure they are equal. // Compare values in both to make sure they are equal.
for (String pathName: paths.values()) { for (String pathName : paths.values()) {
assertTrue(commitPathNames.contains(pathName)); assertTrue(commitPathNames.contains(pathName));
} }
}
private HoodieCleanStat getCleanStat(List<HoodieCleanStat> hoodieCleanStatsTwo, private HoodieCleanStat getCleanStat(List<HoodieCleanStat> hoodieCleanStatsTwo,
String partitionPath) { String partitionPath) {

7
hoodie-common/src/main/java/com/uber/hoodie/common/model/HoodieCommitMetadata.java
View File

@@ -86,10 +86,9 @@ public class HoodieCommitMetadata implements Serializable {
public HashMap<String, String> getFileIdAndFullPaths(String basePath) { public HashMap<String, String> getFileIdAndFullPaths(String basePath) {
HashMap<String, String> fullPaths = new HashMap<>(); HashMap<String, String> fullPaths = new HashMap<>();
HashMap<String, String> relativePaths = getFileIdAndRelativePaths(); for (Map.Entry<String, String> entry: getFileIdAndRelativePaths().entrySet()) {
for (Map.Entry<String, String> entry: relativePaths.entrySet()) { String fullPath = (entry.getValue() != null) ? (new Path(basePath, entry.getValue())).toString() : null;
Path fullPath = new Path(basePath, entry.getValue()); fullPaths.put(entry.getKey(), fullPath);
fullPaths.put(entry.getKey(), fullPath.toString());
} return fullPaths; } return fullPaths;
} }