diff --git a/hoodie-client/src/main/java/com/uber/hoodie/io/HoodieAppendHandle.java b/hoodie-client/src/main/java/com/uber/hoodie/io/HoodieAppendHandle.java index 8ad69ae02..c145ebce4 100644 --- a/hoodie-client/src/main/java/com/uber/hoodie/io/HoodieAppendHandle.java +++ b/hoodie-client/src/main/java/com/uber/hoodie/io/HoodieAppendHandle.java @@ -80,8 +80,6 @@ public class HoodieAppendHandle extends HoodieIOH fileSystemView.getLatestDataFilesForFileId(record.getPartitionPath(), fileId) .findFirst().get().getFileName(); String baseCommitTime = FSUtils.getCommitTime(latestValidFilePath); - Path path = new Path(record.getPartitionPath(), - FSUtils.makeDataFileName(commitTime, TaskContext.getPartitionId(), fileId)); writeStatus.getStat().setPrevCommit(baseCommitTime); writeStatus.setFileId(fileId); writeStatus.setPartitionPath(record.getPartitionPath()); @@ -105,6 +103,8 @@ public class HoodieAppendHandle extends HoodieIOH + " on commit " + commitTime + " on HDFS path " + hoodieTable .getMetaClient().getBasePath() + partitionPath, e); } + Path path = new Path(record.getPartitionPath(), + FSUtils.makeDataFileName(commitTime, TaskContext.getPartitionId(), fileId)); writeStatus.getStat().setPath(path.toString()); } // update the new location of the record, so we know where to find it next diff --git a/hoodie-client/src/main/java/com/uber/hoodie/io/HoodieInsertHandle.java b/hoodie-client/src/main/java/com/uber/hoodie/io/HoodieInsertHandle.java index 8c2eba61d..881fada1f 100644 --- a/hoodie-client/src/main/java/com/uber/hoodie/io/HoodieInsertHandle.java +++ b/hoodie-client/src/main/java/com/uber/hoodie/io/HoodieInsertHandle.java @@ -123,13 +123,12 @@ public class HoodieInsertHandle extends HoodieIOH try { storageWriter.close(); - String relativePath = path.toString().replace(new Path(config.getBasePath()) + "/", ""); - HoodieWriteStat stat = new HoodieWriteStat(); stat.setNumWrites(recordsWritten); stat.setNumDeletes(recordsDeleted); stat.setPrevCommit(HoodieWriteStat.NULL_COMMIT); stat.setFileId(status.getFileId()); + String relativePath = path.toString().replace(new Path(config.getBasePath()) + "/", ""); stat.setPath(relativePath); stat.setTotalWriteBytes(FSUtils.getFileSize(fs, path)); stat.setTotalWriteErrors(status.getFailedRecords().size()); diff --git a/hoodie-client/src/test/java/com/uber/hoodie/TestHoodieClient.java b/hoodie-client/src/test/java/com/uber/hoodie/TestHoodieClient.java index d6b34fda3..61ff10af5 100644 --- a/hoodie-client/src/test/java/com/uber/hoodie/TestHoodieClient.java +++ b/hoodie-client/src/test/java/com/uber/hoodie/TestHoodieClient.java @@ -1137,6 +1137,7 @@ public class TestHoodieClient implements Serializable { List hoodieCleanStatsFour = table.clean(jsc); assertEquals("Must not clean any files" , 0, getCleanStat(hoodieCleanStatsFour, partitionPaths[0]).getSuccessDeleteFiles().size()); assertTrue(HoodieTestUtils.doesDataFileExist(basePath, partitionPaths[0], "002", file3P0C2)); + } @Test public void testKeepLatestCommits() throws IOException { @@ -1335,9 +1336,10 @@ public class TestHoodieClient implements Serializable { inputStream.close(); // Compare values in both to make sure they are equal. - for (String pathName: paths.values()) { + for (String pathName : paths.values()) { assertTrue(commitPathNames.contains(pathName)); } + } private HoodieCleanStat getCleanStat(List hoodieCleanStatsTwo, String partitionPath) { diff --git a/hoodie-common/src/main/java/com/uber/hoodie/common/model/HoodieCommitMetadata.java b/hoodie-common/src/main/java/com/uber/hoodie/common/model/HoodieCommitMetadata.java index 9e2713e78..28d884754 100644 --- a/hoodie-common/src/main/java/com/uber/hoodie/common/model/HoodieCommitMetadata.java +++ b/hoodie-common/src/main/java/com/uber/hoodie/common/model/HoodieCommitMetadata.java @@ -86,10 +86,9 @@ public class HoodieCommitMetadata implements Serializable { public HashMap getFileIdAndFullPaths(String basePath) { HashMap fullPaths = new HashMap<>(); - HashMap relativePaths = getFileIdAndRelativePaths(); - for (Map.Entry entry: relativePaths.entrySet()) { - Path fullPath = new Path(basePath, entry.getValue()); - fullPaths.put(entry.getKey(), fullPath.toString()); + for (Map.Entry entry: getFileIdAndRelativePaths().entrySet()) { + String fullPath = (entry.getValue() != null) ? (new Path(basePath, entry.getValue())).toString() : null; + fullPaths.put(entry.getKey(), fullPath); } return fullPaths; }