FSUtils.getAllPartitionsPaths() works based on .hoodie_partition_metadata
- clean/rollback/write paths covered by existing tests - Snapshot copier fixed to copy metadata file also, and test fixed - Existing tables need to be repaired by addition of metadata, before this can be rolled out
This commit is contained in:
Vinoth Chandar
committed by
vinoth chandar
vinoth chandar
parent
3129770fd0
commit
f9fd16069d
@@ -16,6 +16,7 @@
|
||||
|
||||
package com.uber.hoodie.utilities;
|
||||
|
||||
import com.uber.hoodie.common.HoodieTestDataGenerator;
|
||||
import com.uber.hoodie.common.model.HoodieTestUtils;
|
||||
import com.uber.hoodie.common.util.FSUtils;
|
||||
import org.apache.hadoop.fs.FileSystem;
|
||||
@@ -64,77 +65,79 @@ public class TestHoodieSnapshotCopier {
|
||||
HoodieSnapshotCopier copier = new HoodieSnapshotCopier();
|
||||
copier.snapshot(jsc, basePath, outputPath);
|
||||
|
||||
// Nothing changed except _SUCCESS
|
||||
// Nothing changed; we just bail out
|
||||
assertEquals(fs.listStatus(new Path(basePath)).length, 1);
|
||||
assertTrue(fs.exists(new Path(outputPath + "/_SUCCESS")));
|
||||
assertFalse(fs.exists(new Path(outputPath + "/_SUCCESS")));
|
||||
}
|
||||
|
||||
//TODO - uncomment this after fixing test failures
|
||||
// @Test
|
||||
// public void testSnapshotCopy() throws Exception {
|
||||
// // Generate some commits and corresponding parquets
|
||||
// String commitTime1 = "20160501010101";
|
||||
// String commitTime2 = "20160502020601";
|
||||
// String commitTime3 = "20160506030611";
|
||||
// new File(basePath + "/.hoodie").mkdirs();
|
||||
// new File(basePath + "/.hoodie/hoodie.properties").createNewFile();
|
||||
// // Only first two have commit files
|
||||
// new File(basePath + "/.hoodie/" + commitTime1 + ".commit").createNewFile();
|
||||
// new File(basePath + "/.hoodie/" + commitTime2 + ".commit").createNewFile();
|
||||
// new File(basePath + "/.hoodie/" + commitTime3 + ".inflight").createNewFile();
|
||||
//
|
||||
// // Some parquet files
|
||||
// new File(basePath + "/2016/05/01/").mkdirs();
|
||||
// new File(basePath + "/2016/05/02/").mkdirs();
|
||||
// new File(basePath + "/2016/05/06/").mkdirs();
|
||||
//
|
||||
// // Make commit1
|
||||
// File file11 = new File(basePath + "/2016/05/01/" + FSUtils.makeDataFileName(commitTime1, 1, "id11"));
|
||||
// file11.createNewFile();
|
||||
// File file12 = new File(basePath + "/2016/05/02/" + FSUtils.makeDataFileName(commitTime1, 1, "id12"));
|
||||
// file12.createNewFile();
|
||||
// File file13 = new File(basePath + "/2016/05/06/" + FSUtils.makeDataFileName(commitTime1, 1, "id13"));
|
||||
// file13.createNewFile();
|
||||
//
|
||||
// // Make commit2
|
||||
// File file21 = new File(basePath + "/2016/05/01/" + FSUtils.makeDataFileName(commitTime2, 1, "id21"));
|
||||
// file21.createNewFile();
|
||||
// File file22 = new File(basePath + "/2016/05/02/" + FSUtils.makeDataFileName(commitTime2, 1, "id22"));
|
||||
// file22.createNewFile();
|
||||
// File file23 = new File(basePath + "/2016/05/06/" + FSUtils.makeDataFileName(commitTime2, 1, "id23"));
|
||||
// file23.createNewFile();
|
||||
//
|
||||
// // Make commit3
|
||||
// File file31 = new File(basePath + "/2016/05/01/" + FSUtils.makeDataFileName(commitTime3, 1, "id31"));
|
||||
// file31.createNewFile();
|
||||
// File file32 = new File(basePath + "/2016/05/02/" + FSUtils.makeDataFileName(commitTime3, 1, "id32"));
|
||||
// file32.createNewFile();
|
||||
// File file33 = new File(basePath + "/2016/05/06/" + FSUtils.makeDataFileName(commitTime3, 1, "id33"));
|
||||
// file33.createNewFile();
|
||||
//
|
||||
// // Do a snapshot copy
|
||||
// HoodieSnapshotCopier copier = new HoodieSnapshotCopier();
|
||||
// copier.snapshot(jsc, basePath, outputPath);
|
||||
//
|
||||
// // Check results
|
||||
// assertTrue(fs.exists(new Path(outputPath + "/2016/05/01/" + file11.getName())));
|
||||
// assertTrue(fs.exists(new Path(outputPath + "/2016/05/02/" + file12.getName())));
|
||||
// assertTrue(fs.exists(new Path(outputPath + "/2016/05/06/" + file13.getName())));
|
||||
// assertTrue(fs.exists(new Path(outputPath + "/2016/05/01/" + file21.getName())));
|
||||
// assertTrue(fs.exists(new Path(outputPath + "/2016/05/02/" + file22.getName())));
|
||||
// assertTrue(fs.exists(new Path(outputPath + "/2016/05/06/" + file23.getName())));
|
||||
// assertFalse(fs.exists(new Path(outputPath + "/2016/05/01/" + file31.getName())));
|
||||
// assertFalse(fs.exists(new Path(outputPath + "/2016/05/02/" + file32.getName())));
|
||||
// assertFalse(fs.exists(new Path(outputPath + "/2016/05/06/" + file33.getName())));
|
||||
//
|
||||
// assertTrue(fs.exists(new Path(outputPath + "/.hoodie/" + commitTime1 + ".commit")));
|
||||
// assertTrue(fs.exists(new Path(outputPath + "/.hoodie/" + commitTime2 + ".commit")));
|
||||
// assertFalse(fs.exists(new Path(outputPath + "/.hoodie/" + commitTime3 + ".commit")));
|
||||
// assertFalse(fs.exists(new Path(outputPath + "/.hoodie/" + commitTime3 + ".inflight")));
|
||||
// assertTrue(fs.exists(new Path(outputPath + "/.hoodie/hoodie.properties")));
|
||||
//
|
||||
// assertTrue(fs.exists(new Path(outputPath + "/_SUCCESS")));
|
||||
// }
|
||||
@Test
|
||||
public void testSnapshotCopy() throws Exception {
|
||||
// Generate some commits and corresponding parquets
|
||||
String commitTime1 = "20160501010101";
|
||||
String commitTime2 = "20160502020601";
|
||||
String commitTime3 = "20160506030611";
|
||||
new File(basePath + "/.hoodie").mkdirs();
|
||||
new File(basePath + "/.hoodie/hoodie.properties").createNewFile();
|
||||
// Only first two have commit files
|
||||
new File(basePath + "/.hoodie/" + commitTime1 + ".commit").createNewFile();
|
||||
new File(basePath + "/.hoodie/" + commitTime2 + ".commit").createNewFile();
|
||||
new File(basePath + "/.hoodie/" + commitTime3 + ".inflight").createNewFile();
|
||||
|
||||
// Some parquet files
|
||||
new File(basePath + "/2016/05/01/").mkdirs();
|
||||
new File(basePath + "/2016/05/02/").mkdirs();
|
||||
new File(basePath + "/2016/05/06/").mkdirs();
|
||||
HoodieTestDataGenerator.writePartitionMetadata(fs,
|
||||
new String[] {"2016/05/01", "2016/05/02", "2016/05/06"},
|
||||
basePath);
|
||||
// Make commit1
|
||||
File file11 = new File(basePath + "/2016/05/01/" + FSUtils.makeDataFileName(commitTime1, 1, "id11"));
|
||||
file11.createNewFile();
|
||||
File file12 = new File(basePath + "/2016/05/02/" + FSUtils.makeDataFileName(commitTime1, 1, "id12"));
|
||||
file12.createNewFile();
|
||||
File file13 = new File(basePath + "/2016/05/06/" + FSUtils.makeDataFileName(commitTime1, 1, "id13"));
|
||||
file13.createNewFile();
|
||||
|
||||
// Make commit2
|
||||
File file21 = new File(basePath + "/2016/05/01/" + FSUtils.makeDataFileName(commitTime2, 1, "id21"));
|
||||
file21.createNewFile();
|
||||
File file22 = new File(basePath + "/2016/05/02/" + FSUtils.makeDataFileName(commitTime2, 1, "id22"));
|
||||
file22.createNewFile();
|
||||
File file23 = new File(basePath + "/2016/05/06/" + FSUtils.makeDataFileName(commitTime2, 1, "id23"));
|
||||
file23.createNewFile();
|
||||
|
||||
// Make commit3
|
||||
File file31 = new File(basePath + "/2016/05/01/" + FSUtils.makeDataFileName(commitTime3, 1, "id31"));
|
||||
file31.createNewFile();
|
||||
File file32 = new File(basePath + "/2016/05/02/" + FSUtils.makeDataFileName(commitTime3, 1, "id32"));
|
||||
file32.createNewFile();
|
||||
File file33 = new File(basePath + "/2016/05/06/" + FSUtils.makeDataFileName(commitTime3, 1, "id33"));
|
||||
file33.createNewFile();
|
||||
|
||||
// Do a snapshot copy
|
||||
HoodieSnapshotCopier copier = new HoodieSnapshotCopier();
|
||||
copier.snapshot(jsc, basePath, outputPath);
|
||||
|
||||
// Check results
|
||||
assertTrue(fs.exists(new Path(outputPath + "/2016/05/01/" + file11.getName())));
|
||||
assertTrue(fs.exists(new Path(outputPath + "/2016/05/02/" + file12.getName())));
|
||||
assertTrue(fs.exists(new Path(outputPath + "/2016/05/06/" + file13.getName())));
|
||||
assertTrue(fs.exists(new Path(outputPath + "/2016/05/01/" + file21.getName())));
|
||||
assertTrue(fs.exists(new Path(outputPath + "/2016/05/02/" + file22.getName())));
|
||||
assertTrue(fs.exists(new Path(outputPath + "/2016/05/06/" + file23.getName())));
|
||||
assertFalse(fs.exists(new Path(outputPath + "/2016/05/01/" + file31.getName())));
|
||||
assertFalse(fs.exists(new Path(outputPath + "/2016/05/02/" + file32.getName())));
|
||||
assertFalse(fs.exists(new Path(outputPath + "/2016/05/06/" + file33.getName())));
|
||||
|
||||
assertTrue(fs.exists(new Path(outputPath + "/.hoodie/" + commitTime1 + ".commit")));
|
||||
assertTrue(fs.exists(new Path(outputPath + "/.hoodie/" + commitTime2 + ".commit")));
|
||||
assertFalse(fs.exists(new Path(outputPath + "/.hoodie/" + commitTime3 + ".commit")));
|
||||
assertFalse(fs.exists(new Path(outputPath + "/.hoodie/" + commitTime3 + ".inflight")));
|
||||
assertTrue(fs.exists(new Path(outputPath + "/.hoodie/hoodie.properties")));
|
||||
|
||||
assertTrue(fs.exists(new Path(outputPath + "/_SUCCESS")));
|
||||
}
|
||||
|
||||
@After
|
||||
public void cleanup() {
|
||||
|
||||
Reference in New Issue
Block a user