lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-4469] Flip reuse flag to true in HoodieBackedTableMetadata to improve file listing (#6214)

Browse Source
This commit is contained in:
Y Ethan Guo
2022-07-27 14:04:59 -07:00
committed by GitHub
parent 717f159bfd
commit 924c30c7ea
2 changed files with 17 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files

28
hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieBackedTableMetadata.java
View File

@@ -18,12 +18,6 @@
package org.apache.hudi.client.functional;
import org.apache.avro.Schema;
import org.apache.avro.generic.GenericRecord;
import org.apache.avro.generic.IndexedRecord;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.io.hfile.CacheConfig;
import org.apache.hudi.avro.HoodieAvroUtils;
import org.apache.hudi.avro.model.HoodieMetadataRecord;
import org.apache.hudi.common.config.HoodieMetadataConfig;
@@ -51,13 +45,20 @@ import org.apache.hudi.metadata.HoodieTableMetadataKeyGenerator;
import org.apache.hudi.metadata.MetadataPartitionType;
import org.apache.hudi.table.HoodieSparkTable;
import org.apache.hudi.table.HoodieTable;
import org.apache.avro.Schema;
import org.apache.avro.generic.GenericRecord;
import org.apache.avro.generic.IndexedRecord;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.io.hfile.CacheConfig;
import org.apache.log4j.LogManager;
import org.apache.log4j.Logger;
import org.apache.parquet.avro.AvroSchemaConverter;
import org.apache.parquet.schema.MessageType;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.EnumSource;
import org.junit.jupiter.params.provider.ValueSource;
import java.io.IOException;
import java.util.ArrayList;
@@ -81,23 +82,26 @@ public class TestHoodieBackedTableMetadata extends TestHoodieMetadataBase {
private static final Logger LOG = LogManager.getLogger(TestHoodieBackedTableMetadata.class);
@Test
public void testTableOperations() throws Exception {
@ParameterizedTest
@ValueSource(booleans = {true, false})
public void testTableOperations(boolean reuseReaders) throws Exception {
HoodieTableType tableType = HoodieTableType.COPY_ON_WRITE;
init(tableType);
doWriteInsertAndUpsert(testTable);
// trigger an upsert
doWriteOperation(testTable, "0000003");
verifyBaseMetadataTable();
verifyBaseMetadataTable(reuseReaders);
}
private void doWriteInsertAndUpsert(HoodieTestTable testTable) throws Exception {
doWriteInsertAndUpsert(testTable, "0000001", "0000002", false);
}
private void verifyBaseMetadataTable() throws IOException {
HoodieBackedTableMetadata tableMetadata = new HoodieBackedTableMetadata(context, writeConfig.getMetadataConfig(), writeConfig.getBasePath(), writeConfig.getSpillableMapBasePath(), false);
private void verifyBaseMetadataTable(boolean reuseMetadataReaders) throws IOException {
HoodieBackedTableMetadata tableMetadata = new HoodieBackedTableMetadata(
context, writeConfig.getMetadataConfig(), writeConfig.getBasePath(),
writeConfig.getSpillableMapBasePath(), reuseMetadataReaders);
assertTrue(tableMetadata.enabled());
List<java.nio.file.Path> fsPartitionPaths = testTable.getAllPartitionPaths();
List<String> fsPartitions = new ArrayList<>();

2
hudi-common/src/main/java/org/apache/hudi/metadata/HoodieMetadataFileSystemView.java
View File

@@ -51,7 +51,7 @@ public class HoodieMetadataFileSystemView extends HoodieTableFileSystemView {
HoodieMetadataConfig metadataConfig) {
super(metaClient, visibleActiveTimeline);
this.tableMetadata = HoodieTableMetadata.create(engineContext, metadataConfig, metaClient.getBasePath(),
FileSystemViewStorageConfig.SPILLABLE_DIR.defaultValue());
FileSystemViewStorageConfig.SPILLABLE_DIR.defaultValue(), true);
}
/**