lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-4469] Flip reuse flag to true in HoodieBackedTableMetadata to improve file listing (#6214)

Browse Source
This commit is contained in:
Y Ethan Guo
2022-07-27 14:04:59 -07:00
committed by GitHub
parent 717f159bfd
commit 924c30c7ea
2 changed files with 17 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files

28
hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieBackedTableMetadata.java
View File

@@ -18,12 +18,6 @@
package org.apache.hudi.client.functional; package org.apache.hudi.client.functional;
import org.apache.avro.Schema;
import org.apache.avro.generic.GenericRecord;
import org.apache.avro.generic.IndexedRecord;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.io.hfile.CacheConfig;
import org.apache.hudi.avro.HoodieAvroUtils; import org.apache.hudi.avro.HoodieAvroUtils;
import org.apache.hudi.avro.model.HoodieMetadataRecord; import org.apache.hudi.avro.model.HoodieMetadataRecord;
import org.apache.hudi.common.config.HoodieMetadataConfig; import org.apache.hudi.common.config.HoodieMetadataConfig;
@@ -51,13 +45,20 @@ import org.apache.hudi.metadata.HoodieTableMetadataKeyGenerator;
import org.apache.hudi.metadata.MetadataPartitionType; import org.apache.hudi.metadata.MetadataPartitionType;
import org.apache.hudi.table.HoodieSparkTable; import org.apache.hudi.table.HoodieSparkTable;
import org.apache.hudi.table.HoodieTable; import org.apache.hudi.table.HoodieTable;
import org.apache.avro.Schema;
import org.apache.avro.generic.GenericRecord;
import org.apache.avro.generic.IndexedRecord;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.io.hfile.CacheConfig;
import org.apache.log4j.LogManager; import org.apache.log4j.LogManager;
import org.apache.log4j.Logger; import org.apache.log4j.Logger;
import org.apache.parquet.avro.AvroSchemaConverter; import org.apache.parquet.avro.AvroSchemaConverter;
import org.apache.parquet.schema.MessageType; import org.apache.parquet.schema.MessageType;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.EnumSource; import org.junit.jupiter.params.provider.EnumSource;
import org.junit.jupiter.params.provider.ValueSource;
import java.io.IOException; import java.io.IOException;
import java.util.ArrayList; import java.util.ArrayList;
@@ -81,23 +82,26 @@ public class TestHoodieBackedTableMetadata extends TestHoodieMetadataBase {
private static final Logger LOG = LogManager.getLogger(TestHoodieBackedTableMetadata.class); private static final Logger LOG = LogManager.getLogger(TestHoodieBackedTableMetadata.class);
@Test @ParameterizedTest
public void testTableOperations() throws Exception { @ValueSource(booleans = {true, false})
public void testTableOperations(boolean reuseReaders) throws Exception {
HoodieTableType tableType = HoodieTableType.COPY_ON_WRITE; HoodieTableType tableType = HoodieTableType.COPY_ON_WRITE;
init(tableType); init(tableType);
doWriteInsertAndUpsert(testTable); doWriteInsertAndUpsert(testTable);
// trigger an upsert // trigger an upsert
doWriteOperation(testTable, "0000003"); doWriteOperation(testTable, "0000003");
verifyBaseMetadataTable(); verifyBaseMetadataTable(reuseReaders);
} }
private void doWriteInsertAndUpsert(HoodieTestTable testTable) throws Exception { private void doWriteInsertAndUpsert(HoodieTestTable testTable) throws Exception {
doWriteInsertAndUpsert(testTable, "0000001", "0000002", false); doWriteInsertAndUpsert(testTable, "0000001", "0000002", false);
} }
private void verifyBaseMetadataTable() throws IOException { private void verifyBaseMetadataTable(boolean reuseMetadataReaders) throws IOException {
HoodieBackedTableMetadata tableMetadata = new HoodieBackedTableMetadata(context, writeConfig.getMetadataConfig(), writeConfig.getBasePath(), writeConfig.getSpillableMapBasePath(), false); HoodieBackedTableMetadata tableMetadata = new HoodieBackedTableMetadata(
context, writeConfig.getMetadataConfig(), writeConfig.getBasePath(),
writeConfig.getSpillableMapBasePath(), reuseMetadataReaders);
assertTrue(tableMetadata.enabled()); assertTrue(tableMetadata.enabled());
List<java.nio.file.Path> fsPartitionPaths = testTable.getAllPartitionPaths(); List<java.nio.file.Path> fsPartitionPaths = testTable.getAllPartitionPaths();
List<String> fsPartitions = new ArrayList<>(); List<String> fsPartitions = new ArrayList<>();

2
hudi-common/src/main/java/org/apache/hudi/metadata/HoodieMetadataFileSystemView.java
View File

@@ -51,7 +51,7 @@ public class HoodieMetadataFileSystemView extends HoodieTableFileSystemView {
HoodieMetadataConfig metadataConfig) { HoodieMetadataConfig metadataConfig) {
super(metaClient, visibleActiveTimeline); super(metaClient, visibleActiveTimeline);
this.tableMetadata = HoodieTableMetadata.create(engineContext, metadataConfig, metaClient.getBasePath(), this.tableMetadata = HoodieTableMetadata.create(engineContext, metadataConfig, metaClient.getBasePath(),
FileSystemViewStorageConfig.SPILLABLE_DIR.defaultValue()); FileSystemViewStorageConfig.SPILLABLE_DIR.defaultValue(), true);
} }
/** /**