lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-3200] deprecate hoodie.file.index.enable and unify to use BaseFileOnlyViewRelation to handle (#4798)

Browse Source
This commit is contained in:
Yann Byron
2022-02-15 09:38:01 +08:00
committed by GitHub
parent 0a97a9893a
commit 3b401d839c
8 changed files with 87 additions and 103 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
hudi-common/src/main/java/org/apache/hudi/common/table/TableSchemaResolver.java
View File

@@ -222,14 +222,21 @@ public class TableSchemaResolver {
}
/**
* Gets the schema for a hoodie table in Avro format from the HoodieCommitMetadata of the last commit.
* Gets the schema for a hoodie table in Avro format from the HoodieCommitMetadata of the last commit with valid schema.
*
* @return Avro schema for this table
*/
private Option<Schema> getTableSchemaFromCommitMetadata(boolean includeMetadataFields) {
HoodieTimeline timeline = metaClient.getActiveTimeline().getCommitsTimeline().filterCompletedInstants();
if (timeline.lastInstant().isPresent()) {
return getTableSchemaFromCommitMetadata(timeline.lastInstant().get(), includeMetadataFields);
Option<Pair<HoodieInstant, HoodieCommitMetadata>> instantAndCommitMetadata =
metaClient.getActiveTimeline().getLastCommitMetadataWithValidSchema();
if (instantAndCommitMetadata.isPresent()) {
HoodieCommitMetadata commitMetadata = instantAndCommitMetadata.get().getRight();
String schemaStr = commitMetadata.getMetadata(HoodieCommitMetadata.SCHEMA_KEY);
Schema schema = new Schema.Parser().parse(schemaStr);
if (includeMetadataFields) {
schema = HoodieAvroUtils.addMetadataFields(schema, hasOperationField);
}
return Option.of(schema);
} else {
return Option.empty();
}
@@ -519,7 +526,6 @@ public class TableSchemaResolver {
Schema tableAvroSchema = getTableAvroSchemaFromDataFile();
return tableAvroSchema.getField(HoodieRecord.OPERATION_METADATA_FIELD) != null;
} catch (Exception e) {
LOG.warn("Failed to read operation field from avro schema", e);
return false;
}
}

21
hudi-common/src/main/java/org/apache/hudi/common/table/timeline/HoodieActiveTimeline.java
View File

@@ -24,6 +24,7 @@ import org.apache.hudi.common.table.HoodieTableMetaClient;
import org.apache.hudi.common.table.timeline.HoodieInstant.State;
import org.apache.hudi.common.util.FileIOUtils;
import org.apache.hudi.common.util.Option;
import org.apache.hudi.common.util.StringUtils;
import org.apache.hudi.common.util.ValidationUtils;
import org.apache.hudi.common.util.collection.Pair;
import org.apache.hudi.exception.HoodieIOException;
@@ -259,6 +260,26 @@ public class HoodieActiveTimeline extends HoodieDefaultTimeline {
return readDataFromPath(detailPath);
}
/**
* Get the last instant with valid schema, and convert this to HoodieCommitMetadata
*/
public Option<Pair<HoodieInstant, HoodieCommitMetadata>> getLastCommitMetadataWithValidSchema() {
List<HoodieInstant> completed = getCommitsTimeline().filterCompletedInstants().getInstants()
.sorted(Comparator.comparing(HoodieInstant::getTimestamp).reversed()).collect(Collectors.toList());
for (HoodieInstant instant : completed) {
try {
HoodieCommitMetadata commitMetadata = HoodieCommitMetadata.fromBytes(
getInstantDetails(instant).get(), HoodieCommitMetadata.class);
if (!StringUtils.isNullOrEmpty(commitMetadata.getMetadata(HoodieCommitMetadata.SCHEMA_KEY))) {
return Option.of(Pair.of(instant, commitMetadata));
}
} catch (IOException e) {
LOG.warn("Failed to convert instant to HoodieCommitMetadata: " + instant.toString());
}
}
return Option.empty();
}
/**
* Get the last instant with valid data, and convert this to HoodieCommitMetadata
*/