lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-559] : Make the timeline layout version default to be null version

Browse Source
This commit is contained in:
Balaji Varadarajan
2020-01-19 18:51:56 -08:00
committed by Balaji Varadarajan
parent 7087e7d766
commit ba54a7e973
7 changed files with 28 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableConfig.java
View File

@@ -22,6 +22,7 @@ import org.apache.hudi.common.model.HoodieFileFormat;
import org.apache.hudi.common.model.HoodieTableType;
import org.apache.hudi.common.model.OverwriteWithLatestAvroPayload;
import org.apache.hudi.common.model.TimelineLayoutVersion;
import org.apache.hudi.common.util.Option;
import org.apache.hudi.exception.HoodieIOException;
import org.apache.hadoop.fs.FSDataInputStream;
@@ -144,10 +145,10 @@ public class HoodieTableConfig implements Serializable {
return DEFAULT_TABLE_TYPE;
}
public TimelineLayoutVersion getTimelineLayoutVersion() {
return new TimelineLayoutVersion(Integer.valueOf(props.getProperty(HOODIE_TIMELINE_LAYOUT_VERSION,
String.valueOf(DEFAULT_TIMELINE_LAYOUT_VERSION))));
public Option<TimelineLayoutVersion> getTimelineLayoutVersion() {
return props.containsKey(HOODIE_TIMELINE_LAYOUT_VERSION)
? Option.of(new TimelineLayoutVersion(Integer.valueOf(props.getProperty(HOODIE_TIMELINE_LAYOUT_VERSION))))
: Option.empty();
}
/**

9
hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableMetaClient.java
View File

@@ -117,7 +117,14 @@ public class HoodieTableMetaClient implements Serializable {
TableNotFoundException.checkTableValidity(fs, basePathDir, metaPathDir);
this.tableConfig = new HoodieTableConfig(fs, metaPath, payloadClassName);
this.tableType = tableConfig.getTableType();
this.timelineLayoutVersion = layoutVersion.orElse(tableConfig.getTimelineLayoutVersion());
Option<TimelineLayoutVersion> tableConfigVersion = tableConfig.getTimelineLayoutVersion();
if (layoutVersion.isPresent() && tableConfigVersion.isPresent()) {
// Ensure layout version passed in config is not lower than the one seen in hoodie.properties
Preconditions.checkArgument(layoutVersion.get().compareTo(tableConfigVersion.get()) >= 0,
"Layout Version defined in hoodie properties has higher version (" + tableConfigVersion.get()
+ ") than the one passed in config (" + layoutVersion.get() + ")");
}
this.timelineLayoutVersion = layoutVersion.orElseGet(() -> tableConfig.getTimelineLayoutVersion().get());
this.loadActiveTimelineOnLoad = loadActiveTimelineOnLoad;
LOG.info("Finished Loading Table of type " + tableType + "(version=" + timelineLayoutVersion + ") from " + basePath);
if (loadActiveTimelineOnLoad) {

6
hudi-common/src/test/java/org/apache/hudi/common/table/string/TestHoodieActiveTimeline.java
View File

@@ -37,6 +37,7 @@ import org.junit.rules.ExpectedException;
import java.io.IOException;
import java.util.stream.Stream;
import static org.apache.hudi.common.model.TimelineLayoutVersion.VERSION_0;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
@@ -102,11 +103,14 @@ public class TestHoodieActiveTimeline extends HoodieCommonTestHarness {
timeline.getCommitTimeline().filterPendingExcludingCompaction().getInstants());
// Backwards compatibility testing for reading compaction plans
metaClient = HoodieTableMetaClient.initTableType(metaClient.getHadoopConf(),
metaClient.getBasePath(), metaClient.getTableType(), metaClient.getTableConfig().getTableName(),
metaClient.getArchivePath(), metaClient.getTableConfig().getPayloadClass(), VERSION_0);
HoodieInstant instant6 = new HoodieInstant(State.REQUESTED, HoodieTimeline.COMPACTION_ACTION, "9");
byte[] dummy = new byte[5];
HoodieActiveTimeline oldTimeline = new HoodieActiveTimeline(new HoodieTableMetaClient(metaClient.getHadoopConf(),
metaClient.getBasePath(), true, metaClient.getConsistencyGuardConfig(),
Option.of(new TimelineLayoutVersion(TimelineLayoutVersion.VERSION_0))));
Option.of(new TimelineLayoutVersion(VERSION_0))));
// Old Timeline writes both to aux and timeline folder
oldTimeline.saveToCompactionRequested(instant6, Option.of(dummy));
// Now use latest timeline version