lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

Rebases with latest upstream

Browse Source
This commit is contained in:
Jian Xu
2018-01-18 12:39:30 -08:00
committed by vinoth chandar
parent 363e35bb0f
commit 3736243fb3
2 changed files with 7 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
hoodie-client/src/main/java/com/uber/hoodie/table/HoodieCopyOnWriteTable.java
View File

@@ -591,7 +591,7 @@ public class HoodieCopyOnWriteTable<T extends HoodieRecordPayload> extends Hoodi
.map(writeStatus -> { .map(writeStatus -> {
Tuple2<String, HoodieWriteStat> writeStatTuple2 = (Tuple2<String, HoodieWriteStat>) writeStatus; Tuple2<String, HoodieWriteStat> writeStatTuple2 = (Tuple2<String, HoodieWriteStat>) writeStatus;
HoodieWriteStat writeStat = writeStatTuple2._2(); HoodieWriteStat writeStat = writeStatTuple2._2();
final FileSystem fs = FSUtils.getFs(); final FileSystem fs = getMetaClient().getFs();
final Path finalPath = new Path(config.getBasePath(), writeStat.getPath()); final Path finalPath = new Path(config.getBasePath(), writeStat.getPath());
if (writeStat.getTempPath() != null) { if (writeStat.getTempPath() != null) {
@@ -623,7 +623,7 @@ public class HoodieCopyOnWriteTable<T extends HoodieRecordPayload> extends Hoodi
return; return;
} }
final FileSystem fs = FSUtils.getFs(); final FileSystem fs = getMetaClient().getFs();
final Path temporaryFolder = new Path(config.getBasePath(), final Path temporaryFolder = new Path(config.getBasePath(),
HoodieTableMetaClient.TEMPFOLDER_NAME); HoodieTableMetaClient.TEMPFOLDER_NAME);
try { try {
@@ -635,7 +635,7 @@ public class HoodieCopyOnWriteTable<T extends HoodieRecordPayload> extends Hoodi
List<Tuple2<String, Boolean>> results = jsc List<Tuple2<String, Boolean>> results = jsc
.parallelize(fileStatusesList, config.getFinalizeWriteParallelism()) .parallelize(fileStatusesList, config.getFinalizeWriteParallelism())
.map(fileStatus -> { .map(fileStatus -> {
FileSystem fs1 = FSUtils.getFs(); FileSystem fs1 = getMetaClient().getFs();
boolean success = fs1.delete(fileStatus.getPath(), false); boolean success = fs1.delete(fileStatus.getPath(), false);
logger.info("Deleting file in temporary folder" + fileStatus.getPath() + "\t" logger.info("Deleting file in temporary folder" + fileStatus.getPath() + "\t"
+ success); + success);

10
hoodie-client/src/test/java/com/uber/hoodie/TestHoodieClientOnCopyOnWriteStorage.java
View File

@@ -310,7 +310,6 @@ public class TestHoodieClientOnCopyOnWriteStorage implements Serializable {
.build(); .build();
HoodieWriteClient client = new HoodieWriteClient(jsc, cfg); HoodieWriteClient client = new HoodieWriteClient(jsc, cfg);
HoodieIndex index = HoodieIndex.createIndex(cfg, jsc); HoodieIndex index = HoodieIndex.createIndex(cfg, jsc);
FileSystem fs = FSUtils.getFs();
/** /**
* Write 1 (only inserts) * Write 1 (only inserts)
@@ -328,9 +327,8 @@ public class TestHoodieClientOnCopyOnWriteStorage implements Serializable {
assertPartitionMetadata(HoodieTestDataGenerator.DEFAULT_PARTITION_PATHS, fs); assertPartitionMetadata(HoodieTestDataGenerator.DEFAULT_PARTITION_PATHS, fs);
// verify that there is a commit // verify that there is a commit
HoodieTableMetaClient metaClient = new HoodieTableMetaClient(fs, basePath); HoodieTableMetaClient metaClient = new HoodieTableMetaClient(jsc.hadoopConfiguration(), basePath);
HoodieTimeline timeline = new HoodieActiveTimeline(fs, metaClient.getMetaPath()) HoodieTimeline timeline = new HoodieActiveTimeline(metaClient).getCommitTimeline();
.getCommitTimeline();
assertEquals("Expecting a single commit.", 1, assertEquals("Expecting a single commit.", 1,
timeline.findInstantsAfter("000", Integer.MAX_VALUE).countInstants()); timeline.findInstantsAfter("000", Integer.MAX_VALUE).countInstants());
@@ -366,13 +364,13 @@ public class TestHoodieClientOnCopyOnWriteStorage implements Serializable {
assertNoWriteErrors(statuses); assertNoWriteErrors(statuses);
// verify there are now 2 commits // verify there are now 2 commits
timeline = new HoodieActiveTimeline(fs, metaClient.getMetaPath()).getCommitTimeline(); timeline = new HoodieActiveTimeline(metaClient).getCommitTimeline();
assertEquals("Expecting two commits.", assertEquals("Expecting two commits.",
timeline.findInstantsAfter("000", Integer.MAX_VALUE).countInstants(), 2); timeline.findInstantsAfter("000", Integer.MAX_VALUE).countInstants(), 2);
assertEquals("Latest commit should be 004", timeline.lastInstant().get().getTimestamp(), assertEquals("Latest commit should be 004", timeline.lastInstant().get().getTimestamp(),
newCommitTime); newCommitTime);
metaClient = new HoodieTableMetaClient(fs, basePath); metaClient = new HoodieTableMetaClient(jsc.hadoopConfiguration(), basePath);
table = HoodieTable.getHoodieTable(metaClient, getConfig()); table = HoodieTable.getHoodieTable(metaClient, getConfig());
// Index should be able to locate all updates in correct locations. // Index should be able to locate all updates in correct locations.