[HUDI-1295] Metadata Index - Bloom filter and Column stats index to speed up index lookups (#4352)
* [HUDI-1295] Metadata Index - Bloom filter and Column stats index to speed up index lookups - Today, base files have bloom filter at their footers and index lookups have to load the base file to perform any bloom lookups. Though we have interval tree based file purging, we still end up in significant amount of base file read for the bloom filter for the end index lookups for the keys. This index lookup operation can be made more performant by having all the bloom filters in a new metadata partition and doing pointed lookups based on keys. * [HUDI-1295] Metadata Index - Bloom filter and Column stats index to speed up index lookups - Adding indexing support for clean, restore and rollback operations. Each of these operations will now be converted to index records for bloom filter and column stats additionally. * [HUDI-1295] Metadata Index - Bloom filter and Column stats index to speed up index lookups - Making hoodie key consistent for both column stats and bloom index by including fileId instead of fileName, in both read and write paths. - Performance optimization for looking up records in the metadata table. - Avoiding multi column sorting needed for HoodieBloomMetaIndexBatchCheckFunction * [HUDI-1295] Metadata Index - Bloom filter and Column stats index to speed up index lookups - HoodieBloomMetaIndexBatchCheckFunction cleanup to remove unused classes - Base file checking before reading the file footer for bloom or column stats * [HUDI-1295] Metadata Index - Bloom filter and Column stats index to speed up index lookups - Updating the bloom index and column stats index to have full file name included in the key instead of just file id. - Minor test fixes. * [HUDI-1295] Metadata Index - Bloom filter and Column stats index to speed up index lookups - Fixed flink commit method to handle metadata table all partition update records - TestBloomIndex fixes * [HUDI-1295] Metadata Index - Bloom filter and Column stats index to speed up index lookups - SparkHoodieBloomIndexHelper code simplification for various config modes - Signature change for getBloomFilters() and getColumnStats(). Callers can just pass in interested partition and file names, the index key is then constructed internally based on the passed in parameters. - KeyLookupHandle and KeyLookupResults code refactoring - Metadata schema changes - removed the reserved field * [HUDI-1295] Metadata Index - Bloom filter and Column stats index to speed up index lookups - Removing HoodieColumnStatsMetadata and using HoodieColumnRangeMetadata instead. Fixed the users of the the removed class. * [HUDI-1295] Metadata Index - Bloom filter and Column stats index to speed up index lookups - Extending meta index test to cover deletes, compactions, clean and restore table operations. Also, fixed the getBloomFilters() and getColumnStats() to account for deleted entries. * [HUDI-1295] Metadata Index - Bloom filter and Column stats index to speed up index lookups - Addressing review comments - java doc for new classes, keys sorting for lookup, index methods renaming. * [HUDI-1295] Metadata Index - Bloom filter and Column stats index to speed up index lookups - Consolidated the bloom filter checking for keys in to one HoodieMetadataBloomIndexCheckFunction instead of a spearate batch and lazy mode. Removed all the configs around it. - Made the metadata table partition file group count configurable. - Fixed the HoodieKeyLookupHandle to have auto closable file reader when checking bloom filter and range keys. - Config property renames. Test fixes. * [HUDI-1295] Metadata Index - Bloom filter and Column stats index to speed up index lookups - Enabling column stats indexing for all columns by default - Handling column stat generation errors and test update * [HUDI-1295] Metadata Index - Bloom filter and Column stats index to speed up index lookups - Metadata table partition file group count taken from the slices when the table is bootstrapped. - Prep records for the commit refactored to the base class - HoodieFileReader interface changes for filtering keys - Multi column and data types support for colums stats index * [HUDI-1295] Metadata Index - Bloom filter and Column stats index to speed up index lookups - rebase to latest master and merge fixes for the build and test failures * [HUDI-1295] Metadata Index - Bloom filter and Column stats index to speed up index lookups - Extending the metadata column stats type payload schema to include more statistics about the column ranges to help query integration. * [HUDI-1295] Metadata Index - Bloom filter and Column stats index to speed up index lookups - Addressing review comments
This commit is contained in:
Manoj Govindassamy
committed by
GitHub
GitHub
parent
d681824982
commit
5927bdd1c0
@@ -191,6 +191,8 @@ public class TestHoodieBackedMetadata extends TestHoodieMetadataBase {
|
||||
// trigger couple of upserts
|
||||
doWriteOperation(testTable, "0000005");
|
||||
doWriteOperation(testTable, "0000006");
|
||||
doWriteOperation(testTable, "0000007");
|
||||
doCleanAndValidate(testTable, "0000008", Arrays.asList("0000007"));
|
||||
validateMetadata(testTable, true);
|
||||
}
|
||||
|
||||
@@ -222,7 +224,7 @@ public class TestHoodieBackedMetadata extends TestHoodieMetadataBase {
|
||||
testTable.doWriteOperation("0000003", UPSERT, emptyList(), asList("p1", "p2"), 1, true);
|
||||
syncTableMetadata(writeConfig);
|
||||
|
||||
List<String> partitions = metadataWriter(writeConfig).metadata().getAllPartitionPaths();
|
||||
List<String> partitions = metadataWriter(writeConfig).getTableMetadata().getAllPartitionPaths();
|
||||
assertFalse(partitions.contains(nonPartitionDirectory),
|
||||
"Must not contain the non-partition " + nonPartitionDirectory);
|
||||
assertTrue(partitions.contains("p1"), "Must contain partition p1");
|
||||
@@ -345,6 +347,7 @@ public class TestHoodieBackedMetadata extends TestHoodieMetadataBase {
|
||||
if (tableType == MERGE_ON_READ) {
|
||||
doCompaction(testTable, "0000004");
|
||||
}
|
||||
doCleanAndValidate(testTable, "0000005", Arrays.asList("0000001"));
|
||||
validateMetadata(testTable, emptyList(), true);
|
||||
}
|
||||
|
||||
@@ -380,6 +383,32 @@ public class TestHoodieBackedMetadata extends TestHoodieMetadataBase {
|
||||
assertEquals(tableMetadata.getLatestCompactionTime().get(), "0000003001");
|
||||
}
|
||||
|
||||
@ParameterizedTest
|
||||
@EnumSource(HoodieTableType.class)
|
||||
public void testTableOperationsWithMetadataIndex(HoodieTableType tableType) throws Exception {
|
||||
initPath();
|
||||
HoodieWriteConfig writeConfig = getWriteConfigBuilder(true, true, false)
|
||||
.withIndexConfig(HoodieIndexConfig.newBuilder()
|
||||
.bloomIndexBucketizedChecking(false)
|
||||
.build())
|
||||
.withMetadataConfig(HoodieMetadataConfig.newBuilder()
|
||||
.enable(true)
|
||||
.withMetadataIndexBloomFilter(true)
|
||||
.withMetadataIndexBloomFilterFileGroups(4)
|
||||
.withMetadataIndexColumnStats(true)
|
||||
.withMetadataIndexBloomFilterFileGroups(2)
|
||||
.withMetadataIndexForAllColumns(true)
|
||||
.build())
|
||||
.build();
|
||||
init(tableType, writeConfig);
|
||||
testTableOperationsForMetaIndexImpl(writeConfig);
|
||||
}
|
||||
|
||||
private void testTableOperationsForMetaIndexImpl(final HoodieWriteConfig writeConfig) throws Exception {
|
||||
HoodieSparkEngineContext engineContext = new HoodieSparkEngineContext(jsc);
|
||||
testTableOperationsImpl(engineContext, writeConfig);
|
||||
}
|
||||
|
||||
/**
|
||||
* Tests that virtual key configs are honored in base files after compaction in metadata table.
|
||||
*
|
||||
@@ -619,7 +648,7 @@ public class TestHoodieBackedMetadata extends TestHoodieMetadataBase {
|
||||
// Compaction should not be triggered yet. Let's verify no base file
|
||||
// and few log files available.
|
||||
List<FileSlice> fileSlices = table.getSliceView()
|
||||
.getLatestFileSlices(MetadataPartitionType.FILES.partitionPath()).collect(Collectors.toList());
|
||||
.getLatestFileSlices(MetadataPartitionType.FILES.getPartitionPath()).collect(Collectors.toList());
|
||||
if (fileSlices.isEmpty()) {
|
||||
throw new IllegalStateException("LogFile slices are not available!");
|
||||
}
|
||||
@@ -709,7 +738,7 @@ public class TestHoodieBackedMetadata extends TestHoodieMetadataBase {
|
||||
.withBasePath(metadataMetaClient.getBasePath())
|
||||
.withLogFilePaths(logFilePaths)
|
||||
.withLatestInstantTime(latestCommitTimestamp)
|
||||
.withPartition(MetadataPartitionType.FILES.partitionPath())
|
||||
.withPartition(MetadataPartitionType.FILES.getPartitionPath())
|
||||
.withReaderSchema(schema)
|
||||
.withMaxMemorySizeInBytes(100000L)
|
||||
.withBufferSize(4096)
|
||||
@@ -739,7 +768,7 @@ public class TestHoodieBackedMetadata extends TestHoodieMetadataBase {
|
||||
private void verifyMetadataRecordKeyExcludeFromPayloadBaseFiles(HoodieTable table, boolean enableMetaFields) throws IOException {
|
||||
table.getHoodieView().sync();
|
||||
List<FileSlice> fileSlices = table.getSliceView()
|
||||
.getLatestFileSlices(MetadataPartitionType.FILES.partitionPath()).collect(Collectors.toList());
|
||||
.getLatestFileSlices(MetadataPartitionType.FILES.getPartitionPath()).collect(Collectors.toList());
|
||||
if (!fileSlices.get(0).getBaseFile().isPresent()) {
|
||||
throw new IllegalStateException("Base file not available!");
|
||||
}
|
||||
@@ -1058,10 +1087,20 @@ public class TestHoodieBackedMetadata extends TestHoodieMetadataBase {
|
||||
public void testTableOperationsWithRestore(HoodieTableType tableType) throws Exception {
|
||||
init(tableType);
|
||||
HoodieSparkEngineContext engineContext = new HoodieSparkEngineContext(jsc);
|
||||
HoodieWriteConfig writeConfig = getWriteConfigBuilder(true, true, false)
|
||||
.withRollbackUsingMarkers(false).build();
|
||||
testTableOperationsImpl(engineContext, writeConfig);
|
||||
}
|
||||
|
||||
try (SparkRDDWriteClient client = new SparkRDDWriteClient(engineContext,
|
||||
getWriteConfigBuilder(true, true, false).withRollbackUsingMarkers(false).build())) {
|
||||
|
||||
/**
|
||||
* Test all major table operations with the given table, config and context.
|
||||
*
|
||||
* @param engineContext - Engine context
|
||||
* @param writeConfig - Write config
|
||||
* @throws IOException
|
||||
*/
|
||||
private void testTableOperationsImpl(HoodieSparkEngineContext engineContext, HoodieWriteConfig writeConfig) throws IOException {
|
||||
try (SparkRDDWriteClient client = new SparkRDDWriteClient(engineContext, writeConfig)) {
|
||||
// Write 1 (Bulk insert)
|
||||
String newCommitTime = "0000001";
|
||||
List<HoodieRecord> records = dataGen.generateInserts(newCommitTime, 20);
|
||||
@@ -1738,7 +1777,7 @@ public class TestHoodieBackedMetadata extends TestHoodieMetadataBase {
|
||||
assertTrue(metricsRegistry.getAllCounts().containsKey(HoodieMetadataMetrics.INITIALIZE_STR + ".count"));
|
||||
assertTrue(metricsRegistry.getAllCounts().containsKey(HoodieMetadataMetrics.INITIALIZE_STR + ".totalDuration"));
|
||||
assertTrue(metricsRegistry.getAllCounts().get(HoodieMetadataMetrics.INITIALIZE_STR + ".count") >= 1L);
|
||||
final String prefix = MetadataPartitionType.FILES.partitionPath() + ".";
|
||||
final String prefix = MetadataPartitionType.FILES.getPartitionPath() + ".";
|
||||
assertTrue(metricsRegistry.getAllCounts().containsKey(prefix + HoodieMetadataMetrics.STAT_COUNT_BASE_FILES));
|
||||
assertTrue(metricsRegistry.getAllCounts().containsKey(prefix + HoodieMetadataMetrics.STAT_COUNT_LOG_FILES));
|
||||
assertTrue(metricsRegistry.getAllCounts().containsKey(prefix + HoodieMetadataMetrics.STAT_TOTAL_BASE_FILE_SIZE));
|
||||
@@ -1931,7 +1970,10 @@ public class TestHoodieBackedMetadata extends TestHoodieMetadataBase {
|
||||
// in the .hoodie folder.
|
||||
List<String> metadataTablePartitions = FSUtils.getAllPartitionPaths(engineContext, HoodieTableMetadata.getMetadataTableBasePath(basePath),
|
||||
false, false);
|
||||
assertEquals(MetadataPartitionType.values().length, metadataTablePartitions.size());
|
||||
assertEquals(metadataWriter.getEnabledPartitionTypes().size(), metadataTablePartitions.size());
|
||||
|
||||
final Map<String, MetadataPartitionType> metadataEnabledPartitionTypes = new HashMap<>();
|
||||
metadataWriter.getEnabledPartitionTypes().forEach(e -> metadataEnabledPartitionTypes.put(e.getPartitionPath(), e));
|
||||
|
||||
// Metadata table should automatically compact and clean
|
||||
// versions are +1 as autoclean / compaction happens end of commits
|
||||
@@ -1939,10 +1981,13 @@ public class TestHoodieBackedMetadata extends TestHoodieMetadataBase {
|
||||
HoodieTableFileSystemView fsView = new HoodieTableFileSystemView(metadataMetaClient, metadataMetaClient.getActiveTimeline());
|
||||
metadataTablePartitions.forEach(partition -> {
|
||||
List<FileSlice> latestSlices = fsView.getLatestFileSlices(partition).collect(Collectors.toList());
|
||||
assertTrue(latestSlices.stream().map(FileSlice::getBaseFile).count() <= 1, "Should have a single latest base file");
|
||||
assertTrue(latestSlices.size() <= 1, "Should have a single latest file slice");
|
||||
assertTrue(latestSlices.size() <= numFileVersions, "Should limit file slice to "
|
||||
+ numFileVersions + " but was " + latestSlices.size());
|
||||
assertTrue(latestSlices.stream().map(FileSlice::getBaseFile).count()
|
||||
<= metadataEnabledPartitionTypes.get(partition).getFileGroupCount(), "Should have a single latest base file per file group");
|
||||
assertTrue(latestSlices.size()
|
||||
<= metadataEnabledPartitionTypes.get(partition).getFileGroupCount(), "Should have a single latest file slice per file group");
|
||||
assertTrue(latestSlices.size()
|
||||
<= (numFileVersions * metadataEnabledPartitionTypes.get(partition).getFileGroupCount()), "Should limit file slice to "
|
||||
+ numFileVersions + " per file group, but was " + latestSlices.size());
|
||||
});
|
||||
|
||||
LOG.info("Validation time=" + timer.endTimer());
|
||||
|
||||
@@ -247,7 +247,7 @@ public class TestHoodieBackedTableMetadata extends TestHoodieMetadataBase {
|
||||
// Compaction should not be triggered yet. Let's verify no base file
|
||||
// and few log files available.
|
||||
List<FileSlice> fileSlices = table.getSliceView()
|
||||
.getLatestFileSlices(MetadataPartitionType.FILES.partitionPath()).collect(Collectors.toList());
|
||||
.getLatestFileSlices(MetadataPartitionType.FILES.getPartitionPath()).collect(Collectors.toList());
|
||||
if (fileSlices.isEmpty()) {
|
||||
throw new IllegalStateException("LogFile slices are not available!");
|
||||
}
|
||||
@@ -322,7 +322,7 @@ public class TestHoodieBackedTableMetadata extends TestHoodieMetadataBase {
|
||||
.withBasePath(metadataMetaClient.getBasePath())
|
||||
.withLogFilePaths(logFilePaths)
|
||||
.withLatestInstantTime(latestCommitTimestamp)
|
||||
.withPartition(MetadataPartitionType.FILES.partitionPath())
|
||||
.withPartition(MetadataPartitionType.FILES.getPartitionPath())
|
||||
.withReaderSchema(schema)
|
||||
.withMaxMemorySizeInBytes(100000L)
|
||||
.withBufferSize(4096)
|
||||
@@ -351,7 +351,7 @@ public class TestHoodieBackedTableMetadata extends TestHoodieMetadataBase {
|
||||
private void verifyMetadataRecordKeyExcludeFromPayloadBaseFiles(HoodieTable table) throws IOException {
|
||||
table.getHoodieView().sync();
|
||||
List<FileSlice> fileSlices = table.getSliceView()
|
||||
.getLatestFileSlices(MetadataPartitionType.FILES.partitionPath()).collect(Collectors.toList());
|
||||
.getLatestFileSlices(MetadataPartitionType.FILES.getPartitionPath()).collect(Collectors.toList());
|
||||
if (!fileSlices.get(0).getBaseFile().isPresent()) {
|
||||
throw new IllegalStateException("Base file not available!");
|
||||
}
|
||||
|
||||
@@ -34,6 +34,7 @@ import org.apache.hudi.common.table.timeline.versioning.TimelineLayoutVersion;
|
||||
import org.apache.hudi.common.table.view.FileSystemViewStorageConfig;
|
||||
import org.apache.hudi.common.testutils.HoodieMetadataTestTable;
|
||||
import org.apache.hudi.common.testutils.HoodieTestTable;
|
||||
import org.apache.hudi.common.util.Option;
|
||||
import org.apache.hudi.config.HoodieCompactionConfig;
|
||||
import org.apache.hudi.config.HoodieIndexConfig;
|
||||
import org.apache.hudi.config.HoodieStorageConfig;
|
||||
@@ -84,27 +85,22 @@ public class TestHoodieMetadataBase extends HoodieClientTestHarness {
|
||||
init(tableType, true);
|
||||
}
|
||||
|
||||
public void init(HoodieTableType tableType, HoodieWriteConfig writeConfig) throws IOException {
|
||||
init(tableType, Option.of(writeConfig), true, false, false, false);
|
||||
}
|
||||
|
||||
public void init(HoodieTableType tableType, boolean enableMetadataTable) throws IOException {
|
||||
init(tableType, enableMetadataTable, true, false, false);
|
||||
}
|
||||
|
||||
public void init(HoodieTableType tableType, boolean enableMetadataTable, boolean enableFullScan, boolean enableMetrics, boolean
|
||||
validateMetadataPayloadStateConsistency) throws IOException {
|
||||
this.tableType = tableType;
|
||||
initPath();
|
||||
initSparkContexts("TestHoodieMetadata");
|
||||
initFileSystem();
|
||||
fs.mkdirs(new Path(basePath));
|
||||
initTimelineService();
|
||||
initMetaClient(tableType);
|
||||
initTestDataGenerator();
|
||||
metadataTableBasePath = HoodieTableMetadata.getMetadataTableBasePath(basePath);
|
||||
writeConfig = getWriteConfigBuilder(HoodieFailedWritesCleaningPolicy.EAGER, true, enableMetadataTable, enableMetrics,
|
||||
enableFullScan, true, validateMetadataPayloadStateConsistency).build();
|
||||
initWriteConfigAndMetatableWriter(writeConfig, enableMetadataTable);
|
||||
validateMetadataPayloadStateConsistency) throws IOException {
|
||||
init(tableType, Option.empty(), enableMetadataTable, enableFullScan, enableMetrics,
|
||||
validateMetadataPayloadStateConsistency);
|
||||
}
|
||||
|
||||
public void init(HoodieTableType tableType, HoodieWriteConfig writeConfig) throws IOException {
|
||||
public void init(HoodieTableType tableType, Option<HoodieWriteConfig> writeConfig, boolean enableMetadataTable,
|
||||
boolean enableFullScan, boolean enableMetrics, boolean validateMetadataPayloadStateConsistency) throws IOException {
|
||||
this.tableType = tableType;
|
||||
initPath();
|
||||
initSparkContexts("TestHoodieMetadata");
|
||||
@@ -114,8 +110,12 @@ public class TestHoodieMetadataBase extends HoodieClientTestHarness {
|
||||
initMetaClient(tableType);
|
||||
initTestDataGenerator();
|
||||
metadataTableBasePath = HoodieTableMetadata.getMetadataTableBasePath(basePath);
|
||||
this.writeConfig = writeConfig;
|
||||
initWriteConfigAndMetatableWriter(writeConfig, writeConfig.isMetadataTableEnabled());
|
||||
this.writeConfig = writeConfig.isPresent()
|
||||
? writeConfig.get() : getWriteConfigBuilder(HoodieFailedWritesCleaningPolicy.EAGER, true,
|
||||
enableMetadataTable, enableMetrics, enableFullScan, true,
|
||||
validateMetadataPayloadStateConsistency)
|
||||
.build();
|
||||
initWriteConfigAndMetatableWriter(this.writeConfig, enableMetadataTable);
|
||||
}
|
||||
|
||||
protected void initWriteConfigAndMetatableWriter(HoodieWriteConfig writeConfig, boolean enableMetadataTable) {
|
||||
|
||||
@@ -18,9 +18,13 @@
|
||||
|
||||
package org.apache.hudi.index.bloom;
|
||||
|
||||
import org.apache.avro.Schema;
|
||||
import org.apache.hadoop.fs.Path;
|
||||
import org.apache.hudi.client.functional.TestHoodieMetadataBase;
|
||||
import org.apache.hudi.common.bloom.BloomFilter;
|
||||
import org.apache.hudi.common.bloom.BloomFilterFactory;
|
||||
import org.apache.hudi.common.bloom.BloomFilterTypeCode;
|
||||
import org.apache.hudi.common.config.HoodieMetadataConfig;
|
||||
import org.apache.hudi.common.model.HoodieKey;
|
||||
import org.apache.hudi.common.model.HoodieRecord;
|
||||
import org.apache.hudi.common.table.HoodieTableMetaClient;
|
||||
@@ -32,14 +36,10 @@ import org.apache.hudi.config.HoodieIndexConfig;
|
||||
import org.apache.hudi.config.HoodieWriteConfig;
|
||||
import org.apache.hudi.data.HoodieJavaPairRDD;
|
||||
import org.apache.hudi.data.HoodieJavaRDD;
|
||||
import org.apache.hudi.io.HoodieKeyLookupHandle;
|
||||
import org.apache.hudi.index.HoodieIndexUtils;
|
||||
import org.apache.hudi.table.HoodieSparkTable;
|
||||
import org.apache.hudi.table.HoodieTable;
|
||||
import org.apache.hudi.testutils.HoodieClientTestHarness;
|
||||
import org.apache.hudi.testutils.HoodieSparkWriteableTestTable;
|
||||
|
||||
import org.apache.avro.Schema;
|
||||
import org.apache.hadoop.fs.Path;
|
||||
import org.apache.spark.api.java.JavaPairRDD;
|
||||
import org.apache.spark.api.java.JavaRDD;
|
||||
import org.junit.jupiter.api.AfterEach;
|
||||
@@ -48,6 +48,7 @@ import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.params.ParameterizedTest;
|
||||
import org.junit.jupiter.params.provider.Arguments;
|
||||
import org.junit.jupiter.params.provider.MethodSource;
|
||||
import scala.Tuple2;
|
||||
|
||||
import java.nio.file.Paths;
|
||||
import java.util.Arrays;
|
||||
@@ -59,8 +60,6 @@ import java.util.UUID;
|
||||
import java.util.stream.Collectors;
|
||||
import java.util.stream.Stream;
|
||||
|
||||
import scala.Tuple2;
|
||||
|
||||
import static org.apache.hudi.common.testutils.SchemaTestUtil.getSchemaFromResource;
|
||||
import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
@@ -69,14 +68,14 @@ import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
public class TestHoodieBloomIndex extends HoodieClientTestHarness {
|
||||
public class TestHoodieBloomIndex extends TestHoodieMetadataBase {
|
||||
|
||||
private static final Schema SCHEMA = getSchemaFromResource(TestHoodieBloomIndex.class, "/exampleSchema.avsc", true);
|
||||
private static final String TEST_NAME_WITH_PARAMS = "[{index}] Test with rangePruning={0}, treeFiltering={1}, bucketizedChecking={2}";
|
||||
|
||||
public static Stream<Arguments> configParams() {
|
||||
Object[][] data =
|
||||
new Object[][] {{true, true, true}, {false, true, true}, {true, true, false}, {true, false, true}};
|
||||
new Object[][]{{true, true, true}, {false, true, true}, {true, true, false}, {true, false, true}};
|
||||
return Stream.of(data).map(Arguments::of);
|
||||
}
|
||||
|
||||
@@ -99,6 +98,10 @@ public class TestHoodieBloomIndex extends HoodieClientTestHarness {
|
||||
.withIndexConfig(HoodieIndexConfig.newBuilder().bloomIndexPruneByRanges(rangePruning)
|
||||
.bloomIndexTreebasedFilter(treeFiltering).bloomIndexBucketizedChecking(bucketizedChecking)
|
||||
.bloomIndexKeysPerBucket(2).build())
|
||||
.withMetadataConfig(HoodieMetadataConfig.newBuilder()
|
||||
.withMetadataIndexBloomFilter(false)
|
||||
.withMetadataIndexColumnStats(false)
|
||||
.build())
|
||||
.build();
|
||||
}
|
||||
|
||||
@@ -134,7 +137,7 @@ public class TestHoodieBloomIndex extends HoodieClientTestHarness {
|
||||
new HoodieRecord(new HoodieKey(rowChange4.getRowKey(), rowChange4.getPartitionPath()), rowChange4);
|
||||
|
||||
List<String> partitions = Arrays.asList("2016/01/21", "2016/04/01", "2015/03/12");
|
||||
List<ImmutablePair<String, BloomIndexFileInfo>> filesList = index.loadInvolvedFiles(partitions, context, hoodieTable);
|
||||
List<ImmutablePair<String, BloomIndexFileInfo>> filesList = index.loadColumnRangesFromFiles(partitions, context, hoodieTable);
|
||||
// Still 0, as no valid commit
|
||||
assertEquals(0, filesList.size());
|
||||
|
||||
@@ -143,7 +146,7 @@ public class TestHoodieBloomIndex extends HoodieClientTestHarness {
|
||||
.withInserts("2015/03/12", "3", record1)
|
||||
.withInserts("2015/03/12", "4", record2, record3, record4);
|
||||
|
||||
filesList = index.loadInvolvedFiles(partitions, context, hoodieTable);
|
||||
filesList = index.loadColumnRangesFromFiles(partitions, context, hoodieTable);
|
||||
assertEquals(4, filesList.size());
|
||||
|
||||
if (rangePruning) {
|
||||
@@ -241,9 +244,9 @@ public class TestHoodieBloomIndex extends HoodieClientTestHarness {
|
||||
|
||||
HoodieWriteConfig config = HoodieWriteConfig.newBuilder().withPath(basePath).build();
|
||||
HoodieSparkTable table = HoodieSparkTable.create(config, context, metaClient);
|
||||
HoodieKeyLookupHandle keyHandle = new HoodieKeyLookupHandle<>(config, table, Pair.of(partition, fileId));
|
||||
List<String> results = keyHandle.checkCandidatesAgainstFile(hadoopConf, uuids,
|
||||
new Path(Paths.get(basePath, partition, filename).toString()));
|
||||
List<String> results = HoodieIndexUtils.filterKeysFromFile(
|
||||
new Path(Paths.get(basePath, partition, filename).toString()), uuids, hadoopConf);
|
||||
|
||||
assertEquals(results.size(), 2);
|
||||
assertTrue(results.get(0).equals("1eb5b87a-1feh-4edd-87b4-6ec96dc405a0")
|
||||
|| results.get(1).equals("1eb5b87a-1feh-4edd-87b4-6ec96dc405a0"));
|
||||
|
||||
@@ -109,7 +109,7 @@ public class TestHoodieGlobalBloomIndex extends HoodieClientTestHarness {
|
||||
// intentionally missed the partition "2015/03/12" to see if the GlobalBloomIndex can pick it up
|
||||
List<String> partitions = Arrays.asList("2016/01/21", "2016/04/01");
|
||||
// partitions will NOT be respected by this loadInvolvedFiles(...) call
|
||||
List<Pair<String, BloomIndexFileInfo>> filesList = index.loadInvolvedFiles(partitions, context, hoodieTable);
|
||||
List<Pair<String, BloomIndexFileInfo>> filesList = index.loadColumnRangesFromFiles(partitions, context, hoodieTable);
|
||||
// Still 0, as no valid commit
|
||||
assertEquals(0, filesList.size());
|
||||
|
||||
@@ -118,7 +118,7 @@ public class TestHoodieGlobalBloomIndex extends HoodieClientTestHarness {
|
||||
.withInserts("2015/03/12", "3", record1)
|
||||
.withInserts("2015/03/12", "4", record2, record3, record4);
|
||||
|
||||
filesList = index.loadInvolvedFiles(partitions, context, hoodieTable);
|
||||
filesList = index.loadColumnRangesFromFiles(partitions, context, hoodieTable);
|
||||
assertEquals(4, filesList.size());
|
||||
|
||||
Map<String, BloomIndexFileInfo> filesMap = toFileMap(filesList);
|
||||
|
||||
@@ -59,7 +59,6 @@ import org.apache.hudi.metadata.FileSystemBackedTableMetadata;
|
||||
import org.apache.hudi.metadata.HoodieBackedTableMetadataWriter;
|
||||
import org.apache.hudi.metadata.HoodieTableMetadata;
|
||||
import org.apache.hudi.metadata.HoodieTableMetadataWriter;
|
||||
import org.apache.hudi.metadata.MetadataPartitionType;
|
||||
import org.apache.hudi.metadata.SparkHoodieBackedTableMetadataWriter;
|
||||
import org.apache.hudi.table.HoodieSparkTable;
|
||||
import org.apache.hudi.table.HoodieTable;
|
||||
@@ -680,7 +679,7 @@ public abstract class HoodieClientTestHarness extends HoodieCommonTestHarness im
|
||||
// in the .hoodie folder.
|
||||
List<String> metadataTablePartitions = FSUtils.getAllPartitionPaths(engineContext, HoodieTableMetadata.getMetadataTableBasePath(basePath),
|
||||
false, false);
|
||||
Assertions.assertEquals(MetadataPartitionType.values().length, metadataTablePartitions.size());
|
||||
Assertions.assertEquals(metadataWriter.getEnabledPartitionTypes().size(), metadataTablePartitions.size());
|
||||
|
||||
// Metadata table should automatically compact and clean
|
||||
// versions are +1 as autoClean / compaction happens end of commits
|
||||
|
||||
Reference in New Issue
Block a user