lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

Correct clean bug that causes clean failure when partitionPaths are empty

Browse Source
This commit is contained in:
Kaushik Devarajaiah
2017-06-20 00:16:54 -07:00
committed by prazanna
parent 7ef76a4de0
commit 3aa8083913
2 changed files with 24 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
hoodie-client/src/main/java/com/uber/hoodie/table/HoodieCopyOnWriteTable.java
View File

@@ -492,21 +492,19 @@ public class HoodieCopyOnWriteTable<T extends HoodieRecordPayload> extends Hoodi
*/ */
@Override @Override
public List<HoodieCleanStat> clean(JavaSparkContext jsc) { public List<HoodieCleanStat> clean(JavaSparkContext jsc) {
List<HoodieCleanStat> partitionCleanStats;
try { try {
List<String> partitionsToClean = List<String> partitionsToClean =
FSUtils.getAllPartitionPaths(getFs(), getMetaClient().getBasePath(), config.shouldAssumeDatePartitioning()); FSUtils.getAllPartitionPaths(getFs(), getMetaClient().getBasePath(), config.shouldAssumeDatePartitioning());
logger.info("Partitions to clean up : " + partitionsToClean + ", with policy " + config logger.info("Partitions to clean up : " + partitionsToClean + ", with policy " + config
.getCleanerPolicy()); .getCleanerPolicy());
partitionCleanStats = cleanPartitionPaths(partitionsToClean, jsc);
if (partitionsToClean.isEmpty()) { if (partitionsToClean.isEmpty()) {
logger.info("Nothing to clean here mom. It is already clean"); logger.info("Nothing to clean here mom. It is already clean");
return partitionCleanStats; return Collections.emptyList();
} }
return cleanPartitionPaths(partitionsToClean, jsc);
} catch (IOException e) { } catch (IOException e) {
throw new HoodieIOException("Failed to clean up after commit", e); throw new HoodieIOException("Failed to clean up after commit", e);
} }
return partitionCleanStats;
} }
private static class PartitionCleanStat implements Serializable { private static class PartitionCleanStat implements Serializable {

22
hoodie-client/src/test/java/com/uber/hoodie/TestHoodieClient.java
View File

@@ -1225,6 +1225,28 @@ public class TestHoodieClient implements Serializable {
assertTrue(HoodieTestUtils.doesDataFileExist(basePath, partitionPaths[0], "001", file2P0C1)); assertTrue(HoodieTestUtils.doesDataFileExist(basePath, partitionPaths[0], "001", file2P0C1));
} }
@Test
public void testCleaningWithZeroPartitonPaths() throws IOException {
HoodieWriteConfig config = HoodieWriteConfig.newBuilder().withPath(basePath)
.withAssumeDatePartitioning(true)
.withCompactionConfig(HoodieCompactionConfig.newBuilder()
.withCleanerPolicy(HoodieCleaningPolicy.KEEP_LATEST_COMMITS)
.retainCommits(2).build()).build();
// Make a commit, although there are no partitionPaths.
// Example use-case of this is when a client wants to create a table
// with just some commit metadata, but no data/partitionPaths.
HoodieTestUtils.createCommitFiles(basePath, "000");
HoodieTable table = HoodieTable
.getHoodieTable(new HoodieTableMetaClient(FSUtils.getFs(), config.getBasePath(), true),
config);
List<HoodieCleanStat> hoodieCleanStatsOne = table.clean(jsc);
assertTrue("HoodieCleanStats should be empty for a table with empty partitionPaths",
hoodieCleanStatsOne.isEmpty());
}
@Test @Test
public void testCleaningSkewedPartitons() throws IOException { public void testCleaningSkewedPartitons() throws IOException {
HoodieWriteConfig config = HoodieWriteConfig.newBuilder().withPath(basePath) HoodieWriteConfig config = HoodieWriteConfig.newBuilder().withPath(basePath)