lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-756] Organize Cleaning Action execution into a single package in hudi-client (#1485)

Browse Source 
- Introduced a thin abstraction ActionExecutor, that all actions will implement
- Pulled cleaning code from table, writeclient into a single package
- CleanHelper is now CleanPlanner, HoodieCleanClient is no longer around
- Minor refactor of HoodieTable factory method
- HoodieTable.create() methods with and without metaclient passed in
- HoodieTable constructor now does not do a redundant instantiation
- Fixed existing unit tests to work at the HoodieWriteClient level
This commit is contained in:
vinoth chandar
2020-04-04 00:07:34 -07:00
committed by GitHub
parent 575d87cf7d
commit eaf6cc2d90
31 changed files with 585 additions and 555 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
hudi-common/src/main/java/org/apache/hudi/common/util/CleanerUtils.java
View File

@@ -39,8 +39,9 @@ public class CleanerUtils {
public static final Integer CLEAN_METADATA_VERSION_2 = CleanV2MigrationHandler.VERSION;
public static final Integer LATEST_CLEAN_METADATA_VERSION = CLEAN_METADATA_VERSION_2;
public static HoodieCleanMetadata convertCleanMetadata(HoodieTableMetaClient metaClient,
String startCleanTime, Option<Long> durationInMs, List<HoodieCleanStat> cleanStats) {
public static HoodieCleanMetadata convertCleanMetadata(String startCleanTime,
Option<Long> durationInMs,
List<HoodieCleanStat> cleanStats) {
Map<String, HoodieCleanPartitionMetadata> partitionMetadataMap = new HashMap<>();
int totalDeleted = 0;
String earliestCommitToRetain = null;

2
hudi-common/src/test/java/org/apache/hudi/common/model/HoodieTestUtils.java
View File

@@ -328,7 +328,7 @@ public class HoodieTestUtils {
// Create the clean metadata
HoodieCleanMetadata cleanMetadata =
CleanerUtils.convertCleanMetadata(metaClient, instantTime, Option.of(0L), Collections.singletonList(cleanStats));
CleanerUtils.convertCleanMetadata(instantTime, Option.of(0L), Collections.singletonList(cleanStats));
// Write empty clean metadata
os.write(TimelineMetadataUtils.serializeCleanMetadata(cleanMetadata).get());
}

3
hudi-common/src/test/java/org/apache/hudi/common/table/view/TestIncrementalFSViewSync.java
View File

@@ -414,8 +414,7 @@ public class TestIncrementalFSViewSync extends HoodieCommonTestHarness {
HoodieInstant cleanInflightInstant = new HoodieInstant(true, HoodieTimeline.CLEAN_ACTION, cleanInstant);
metaClient.getActiveTimeline().createNewInstant(cleanInflightInstant);
HoodieCleanMetadata cleanMetadata = CleanerUtils
.convertCleanMetadata(metaClient, cleanInstant, Option.empty(), cleanStats);
HoodieCleanMetadata cleanMetadata = CleanerUtils.convertCleanMetadata(cleanInstant, Option.empty(), cleanStats);
metaClient.getActiveTimeline().saveAsComplete(cleanInflightInstant,
TimelineMetadataUtils.serializeCleanMetadata(cleanMetadata));
}