lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

Add FinalizeWrite in HoodieCreateHandle for COW tables

Browse Source
This commit is contained in:
Jian Xu
2017-11-29 16:59:28 -08:00
committed by vinoth chandar
parent e10100fe32
commit c874248f23
13 changed files with 318 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
hoodie-common/src/main/java/com/uber/hoodie/common/model/HoodieWriteStat.java
View File

@@ -70,6 +70,12 @@ public class HoodieWriteStat implements Serializable {
*/
private long totalWriteErrors;
/**
* Relative path to the temporary file from the base path.
*/
@Nullable
private String tempPath;
/**
* Following properties are associated only with the result of a Compaction Operation
*/
@@ -198,11 +204,20 @@ public class HoodieWriteStat implements Serializable {
this.totalRecordsToBeUpdate = totalRecordsToBeUpdate;
}
public void setTempPath(String tempPath) {
this.tempPath = tempPath;
}
public String getTempPath() {
return this.tempPath;
}
@Override
public String toString() {
return new StringBuilder()
.append("HoodieWriteStat {")
.append("path=" + path)
.append(", tempPath=" + tempPath)
.append(", prevCommit='" + prevCommit + '\'')
.append(", numWrites=" + numWrites)
.append(", numDeletes=" + numDeletes)

1
hoodie-common/src/main/java/com/uber/hoodie/common/table/HoodieTableMetaClient.java
View File

@@ -50,6 +50,7 @@ public class HoodieTableMetaClient implements Serializable {
private final transient static Logger log = LogManager.getLogger(HoodieTableMetaClient.class);
public static String METAFOLDER_NAME = ".hoodie";
public static String TEMPFOLDER_NAME = METAFOLDER_NAME + File.separator + ".temp";
private String basePath;
private transient FileSystem fs;

4
hoodie-common/src/main/java/com/uber/hoodie/common/util/FSUtils.java
View File

@@ -94,6 +94,10 @@ public class FSUtils {
return String.format("%s_%d_%s.parquet", fileId, taskPartitionId, commitTime);
}
public static String makeTempDataFileName(String partitionPath, String commitTime, int taskPartitionId, String fileId, int stageId, long taskAttemptId) {
return String.format("%s_%s_%d_%s_%d_%d.parquet", partitionPath.replace("/", "-"), fileId, taskPartitionId, commitTime, stageId, taskAttemptId);
}
public static String maskWithoutFileId(String commitTime, int taskPartitionId) {
return String.format("*_%s_%s.parquet", taskPartitionId, commitTime);
}

12
hoodie-common/src/test/java/com/uber/hoodie/common/util/TestFSUtils.java
View File

@@ -43,6 +43,18 @@ public class TestFSUtils {
.equals(fileName + "_" + taskPartitionId + "_" + commitTime + ".parquet"));
}
@Test
public void testMakeTempDataFileName() {
String commitTime = new SimpleDateFormat("yyyyMMddHHmmss").format(new Date());
String partitionPath = "2017/12/31";
int taskPartitionId = Integer.MAX_VALUE;
int stageId = Integer.MAX_VALUE;
long taskAttemptId = Long.MAX_VALUE;
String fileName = UUID.randomUUID().toString();
assertTrue(FSUtils.makeTempDataFileName(partitionPath, commitTime, taskPartitionId, fileName, stageId, taskAttemptId)
.equals(partitionPath.replace("/", "-") + "_" + fileName + "_" + taskPartitionId + "_" + commitTime + "_" + stageId + "_" + taskAttemptId + ".parquet"));
}
@Test
public void testMaskFileName() {
String commitTime = new SimpleDateFormat("yyyyMMddHHmmss").format(new Date());