lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-1381] Schedule compaction based on time elapsed (#2260)

Browse Source 
- introduce configs to control how compaction is triggered
- Compaction can be triggered using time, number of delta commits and/or combinations
- Default behaviour remains the same.
This commit is contained in:
Karl_Wang
2021-02-17 23:44:53 +08:00
committed by GitHub
parent c4bbcb7f0e
commit 9431aabfab
7 changed files with 354 additions and 46 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
hudi-common/src/main/java/org/apache/hudi/common/table/timeline/HoodieActiveTimeline.java
View File

@@ -81,10 +81,18 @@ public class HoodieActiveTimeline extends HoodieDefaultTimeline {
* Ensures each instant time is atleast 1 second apart since we create instant times at second granularity
*/
public static String createNewInstantTime() {
return createNewInstantTime(0);
}
/**
* Returns next instant time that adds N milliseconds in the {@link #COMMIT_FORMATTER} format.
* Ensures each instant time is atleast 1 second apart since we create instant times at second granularity
*/
public static String createNewInstantTime(long milliseconds) {
return lastInstantTime.updateAndGet((oldVal) -> {
String newCommitTime;
do {
newCommitTime = HoodieActiveTimeline.COMMIT_FORMATTER.format(new Date());
newCommitTime = HoodieActiveTimeline.COMMIT_FORMATTER.format(new Date(System.currentTimeMillis() + milliseconds));
} while (HoodieTimeline.compareTimestamps(newCommitTime, LESSER_THAN_OR_EQUALS, oldVal));
return newCommitTime;
});

30
hudi-common/src/main/java/org/apache/hudi/exception/HoodieCompactException.java Normal file
View File

@@ -0,0 +1,30 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hudi.exception;
public class HoodieCompactException extends HoodieException {
public HoodieCompactException(String msg) {
super(msg);
}
public HoodieCompactException(String msg, Throwable e) {
super(msg, e);
}
}