lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-2472] Fix few Cleaner tests with metadata table enabled (#3825)

Browse Source
This commit is contained in:
Manoj Govindassamy
2021-10-20 15:57:00 -07:00
committed by GitHub
parent e355ab52db
commit b68c5a68f9
6 changed files with 195 additions and 62 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/transaction/TestSimpleConcurrentFileWritesConflictResolutionStrategy.java
View File

@@ -314,7 +314,7 @@ public class TestSimpleConcurrentFileWritesConflictResolutionStrategy extends Ho
commitMetadata.addWriteStat(HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH, writeStat);
commitMetadata.setOperationType(WriteOperationType.INSERT);
HoodieTestTable.of(metaClient)
.addCommit(instantTime, commitMetadata)
.addCommit(instantTime, Option.of(commitMetadata))
.withBaseFilesInPartition(HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH, fileId1, fileId2);
}
@@ -362,7 +362,7 @@ public class TestSimpleConcurrentFileWritesConflictResolutionStrategy extends Ho
writeStat.setFileId("file-1");
commitMetadata.addWriteStat(HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH, writeStat);
HoodieTestTable.of(metaClient)
.addCommit(instantTime, commitMetadata)
.addCommit(instantTime, Option.of(commitMetadata))
.withBaseFilesInPartition(HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH, fileId1, fileId2);
}

24
hudi-client/hudi-client-common/src/test/java/org/apache/hudi/common/testutils/HoodieMetadataTestTable.java
View File

@@ -27,6 +27,7 @@ import org.apache.hudi.common.model.HoodieReplaceCommitMetadata;
import org.apache.hudi.common.model.WriteOperationType;
import org.apache.hudi.common.table.HoodieTableMetaClient;
import org.apache.hudi.common.util.Option;
import org.apache.hudi.common.util.collection.Pair;
import org.apache.hudi.metadata.HoodieTableMetadataWriter;
import org.apache.hadoop.fs.FileSystem;
@@ -40,7 +41,7 @@ import java.util.Map;
*/
public class HoodieMetadataTestTable extends HoodieTestTable {
private HoodieTableMetadataWriter writer;
private final HoodieTableMetadataWriter writer;
protected HoodieMetadataTestTable(String basePath, FileSystem fs, HoodieTableMetaClient metaClient, HoodieTableMetadataWriter writer) {
super(basePath, fs, metaClient);
@@ -56,11 +57,25 @@ public class HoodieMetadataTestTable extends HoodieTestTable {
return new HoodieMetadataTestTable(metaClient.getBasePath(), metaClient.getRawFs(), metaClient, writer);
}
/**
* Add commits to the requested partitions and update metadata table.
*
* @param commitTime - Commit time for the operation
* @param operationType - Operation type
* @param newPartitionsToAdd - New partitions to add for the operation
* @param partitionToFilesNameLengthMap - Map of partition names to its list of files name and length pair
* @param bootstrap - Whether bootstrapping needed for the operation
* @param createInflightCommit - Whether in flight commit needed for the operation
* @return Commit metadata for the commit operation performed.
* @throws Exception
*/
@Override
public HoodieCommitMetadata doWriteOperation(String commitTime, WriteOperationType operationType,
List<String> newPartitionsToAdd, List<String> partitions,
int filesPerPartition, boolean bootstrap, boolean createInflightCommit) throws Exception {
HoodieCommitMetadata commitMetadata = super.doWriteOperation(commitTime, operationType, newPartitionsToAdd, partitions, filesPerPartition, bootstrap, createInflightCommit);
List<String> newPartitionsToAdd,
Map<String, List<Pair<String, Integer>>> partitionToFilesNameLengthMap,
boolean bootstrap, boolean createInflightCommit) throws Exception {
HoodieCommitMetadata commitMetadata = super.doWriteOperation(commitTime, operationType, newPartitionsToAdd,
partitionToFilesNameLengthMap, bootstrap, createInflightCommit);
if (writer != null && !createInflightCommit) {
writer.update(commitMetadata, commitTime);
}
@@ -140,4 +155,5 @@ public class HoodieMetadataTestTable extends HoodieTestTable {
}
return this;
}
}

4
hudi-client/hudi-client-common/src/test/java/org/apache/hudi/utils/TestMetadataConversionUtils.java
View File

@@ -176,7 +176,7 @@ public class TestMetadataConversionUtils extends HoodieCommonTestHarness {
commitMetadata.setOperationType(WriteOperationType.COMPACT);
commitMetadata.setCompacted(true);
HoodieTestTable.of(metaClient)
.addCommit(instantTime, commitMetadata)
.addCommit(instantTime, Option.of(commitMetadata))
.withBaseFilesInPartition(HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH, fileId1, fileId2);
}
@@ -206,7 +206,7 @@ public class TestMetadataConversionUtils extends HoodieCommonTestHarness {
commitMetadata.addMetadata("test", "test");
commitMetadata.setOperationType(WriteOperationType.INSERT);
HoodieTestTable.of(metaClient)
.addCommit(instantTime, commitMetadata)
.addCommit(instantTime, Option.of(commitMetadata))
.withBaseFilesInPartition(HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH, fileId1, fileId2);
}