1
0

Incorporating code review feedback for finalizeWrite for COW #2

This commit is contained in:
Jian Xu
2017-12-14 13:36:32 -08:00
committed by vinoth chandar
parent 2fe4fef625
commit 37f2cdd7e4
8 changed files with 79 additions and 34 deletions

View File

@@ -848,7 +848,6 @@ public class HoodieWriteClient<T extends HoodieRecordPayload> implements Seriali
String commitActionType = table.getCommitActionType();
activeTimeline.createInflight(
new HoodieInstant(true, commitActionType, commitTime));
table.initializeFinalizeWrite();
}
/**

View File

@@ -130,11 +130,7 @@ public class HoodieCreateHandle<T extends HoodieRecordPayload> extends HoodieIOH
stat.setNumDeletes(recordsDeleted);
stat.setPrevCommit(HoodieWriteStat.NULL_COMMIT);
stat.setFileId(status.getFileId());
String relativePath = path.toString().replace(new Path(config.getBasePath()) + "/", "");
stat.setPath(relativePath);
if (tempPath != null) {
stat.setTempPath(tempPath.toString().replace(new Path(config.getBasePath()) + "/", ""));
}
stat.setPaths(new Path(config.getBasePath()), path, tempPath);
stat.setTotalWriteBytes(FSUtils.getFileSize(fs, getStorageWriterPath()));
stat.setTotalWriteErrors(status.getFailedRecords().size());
status.setStat(stat);

View File

@@ -576,24 +576,6 @@ public class HoodieCopyOnWriteTable<T extends HoodieRecordPayload> extends Hoodi
return stats;
}
@Override
public void initializeFinalizeWrite() {
if (!config.shouldUseTempFolderForCopyOnWrite()) {
return;
}
// create temporary folder if needed
final FileSystem fs = FSUtils.getFs();
final Path temporaryFolder = new Path(config.getBasePath(), HoodieTableMetaClient.TEMPFOLDER_NAME);
try {
if (!fs.exists(temporaryFolder)) {
fs.mkdirs(temporaryFolder);
}
} catch (IOException e) {
throw new HoodieIOException("Failed to create temporary folder: " + temporaryFolder);
}
}
@Override
@SuppressWarnings("unchecked")
public Optional<Integer> finalizeWrite(JavaSparkContext jsc, List writeStatuses) {

View File

@@ -250,11 +250,6 @@ public class HoodieMergeOnReadTable<T extends HoodieRecordPayload> extends
return allRollbackStats;
}
@Override
public void initializeFinalizeWrite() {
// do nothing for MOR tables
}
@Override
public Optional<Integer> finalizeWrite(JavaSparkContext jsc, List writeStatuses) {
// do nothing for MOR tables

View File

@@ -273,11 +273,6 @@ public abstract class HoodieTable<T extends HoodieRecordPayload> implements Seri
public abstract List<HoodieRollbackStat> rollback(JavaSparkContext jsc, List<String> commits)
throws IOException;
/**
* Initialize resources needed for finalize write.
*/
public abstract void initializeFinalizeWrite();
/**
* Finalize the written data files
*