lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-2744] Fix parsing of metadadata table compaction timestamp when metrics are enabled (#3976)

Browse Source
This commit is contained in:
Sivabalan Narayanan
2021-11-15 07:27:35 -05:00
committed by GitHub
parent 3c4319729c
commit 53d2d6ae24
5 changed files with 59 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
hudi-common/src/main/java/org/apache/hudi/common/table/timeline/HoodieInstantTimeGenerator.java
View File

@@ -18,6 +18,7 @@
package org.apache.hudi.common.table.timeline;
import java.text.ParseException;
import java.time.Instant;
import java.time.LocalDateTime;
import java.time.ZoneId;
@@ -33,6 +34,7 @@ import java.util.concurrent.atomic.AtomicReference;
public class HoodieInstantTimeGenerator {
// Format of the timestamp used for an Instant
private static final String INSTANT_TIMESTAMP_FORMAT = "yyyyMMddHHmmss";
private static final int INSTANT_TIMESTAMP_FORMAT_LENGTH = INSTANT_TIMESTAMP_FORMAT.length();
// Formatter to generate Instant timestamps
private static DateTimeFormatter INSTANT_TIME_FORMATTER = DateTimeFormatter.ofPattern(INSTANT_TIMESTAMP_FORMAT);
// The last Instant timestamp generated
@@ -56,7 +58,7 @@ public class HoodieInstantTimeGenerator {
});
}
public static Date parseInstantTime(String timestamp) {
public static Date parseInstantTime(String timestamp) throws ParseException {
try {
LocalDateTime dt = LocalDateTime.parse(timestamp, INSTANT_TIME_FORMATTER);
return Date.from(dt.atZone(ZoneId.systemDefault()).toInstant());
@@ -65,7 +67,11 @@ public class HoodieInstantTimeGenerator {
if (timestamp.equals(ALL_ZERO_TIMESTAMP)) {
return new Date(0);
}
// compaction and cleaning in metadata has special format. handling it by trimming extra chars and treating it with secs granularity
if (timestamp.length() > INSTANT_TIMESTAMP_FORMAT_LENGTH) {
LocalDateTime dt = LocalDateTime.parse(timestamp.substring(0, INSTANT_TIMESTAMP_FORMAT_LENGTH), INSTANT_TIME_FORMATTER);
return Date.from(dt.atZone(ZoneId.systemDefault()).toInstant());
}
throw e;
}
}

3
hudi-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadata.java
View File

@@ -184,8 +184,9 @@ public class HoodieBackedTableMetadata extends BaseTableMetadata {
HoodieRecord<HoodieMetadataPayload> hoodieRecord = null;
// Retrieve record from base file
if (baseFileReader != null) {
HoodieTimer readTimer = new HoodieTimer().startTimer();
HoodieTimer readTimer = new HoodieTimer();
for (String key : keys) {
readTimer.startTimer();
Option<GenericRecord> baseRecord = baseFileReader.getRecordByKey(key);
if (baseRecord.isPresent()) {
hoodieRecord = metadataTableConfig.populateMetaFields()