1
0

Add FinalizeWrite in HoodieCreateHandle for COW tables

This commit is contained in:
Jian Xu
2017-11-29 16:59:28 -08:00
committed by vinoth chandar
parent e10100fe32
commit c874248f23
13 changed files with 318 additions and 7 deletions

View File

@@ -70,6 +70,12 @@ public class HoodieWriteStat implements Serializable {
*/
private long totalWriteErrors;
/**
* Relative path to the temporary file from the base path.
*/
@Nullable
private String tempPath;
/**
* Following properties are associated only with the result of a Compaction Operation
*/
@@ -198,11 +204,20 @@ public class HoodieWriteStat implements Serializable {
this.totalRecordsToBeUpdate = totalRecordsToBeUpdate;
}
public void setTempPath(String tempPath) {
this.tempPath = tempPath;
}
public String getTempPath() {
return this.tempPath;
}
@Override
public String toString() {
return new StringBuilder()
.append("HoodieWriteStat {")
.append("path=" + path)
.append(", tempPath=" + tempPath)
.append(", prevCommit='" + prevCommit + '\'')
.append(", numWrites=" + numWrites)
.append(", numDeletes=" + numDeletes)

View File

@@ -50,6 +50,7 @@ public class HoodieTableMetaClient implements Serializable {
private final transient static Logger log = LogManager.getLogger(HoodieTableMetaClient.class);
public static String METAFOLDER_NAME = ".hoodie";
public static String TEMPFOLDER_NAME = METAFOLDER_NAME + File.separator + ".temp";
private String basePath;
private transient FileSystem fs;

View File

@@ -94,6 +94,10 @@ public class FSUtils {
return String.format("%s_%d_%s.parquet", fileId, taskPartitionId, commitTime);
}
public static String makeTempDataFileName(String partitionPath, String commitTime, int taskPartitionId, String fileId, int stageId, long taskAttemptId) {
return String.format("%s_%s_%d_%s_%d_%d.parquet", partitionPath.replace("/", "-"), fileId, taskPartitionId, commitTime, stageId, taskAttemptId);
}
public static String maskWithoutFileId(String commitTime, int taskPartitionId) {
return String.format("*_%s_%s.parquet", taskPartitionId, commitTime);
}