lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-4200] Fixing sorting of keys fetched from metadata table (#5773)

Browse Source 
- Key fetched from metadata table especially from base file reader is not sorted. and hence may result in throwing NPE (key prefix search) or unnecessary seeks to starting of Hfile (full key look ups). Fixing the same in this patch. This is not an issue with log blocks, since sorting is taking care within HoodieHfileDataBlock.
- Commit where the sorting was mistakenly reverted [HUDI-3760] Adding capability to fetch Metadata Records by prefix  #5208
This commit is contained in:
Sivabalan Narayanan
2022-06-07 08:19:52 -04:00
committed by GitHub
parent 4f5cad8029
commit f85cd9b16d
4 changed files with 27 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
hudi-client/hudi-client-common/src/test/java/org/apache/hudi/io/storage/TestHoodieHFileReaderWriter.java
View File

@@ -50,6 +50,7 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
@@ -316,15 +317,20 @@ public class TestHoodieHFileReaderWriter extends TestHoodieReaderWriterBase {
assertEquals(expectedKey50and0s, recordsByPrefix);
// filter for "key1" and "key0" : entries from 'key10 to key19' and 'key00 to key09' should be matched.
List<GenericRecord> expectedKey1sand0s = expectedKey1s;
expectedKey1sand0s.addAll(allRecords.stream()
.filter(entry -> (entry.get("_row_key").toString()).contains("key0"))
.collect(Collectors.toList()));
List<GenericRecord> expectedKey1sand0s = allRecords.stream()
.filter(entry -> (entry.get("_row_key").toString()).contains("key1") || (entry.get("_row_key").toString()).contains("key0"))
.collect(Collectors.toList());
iterator =
hfileReader.getRecordsByKeyPrefixIterator(Arrays.asList("key1", "key0"), avroSchema);
recordsByPrefix =
StreamSupport.stream(Spliterators.spliteratorUnknownSize(iterator, Spliterator.ORDERED), false)
.collect(Collectors.toList());
Collections.sort(recordsByPrefix, new Comparator<GenericRecord>() {
@Override
public int compare(GenericRecord o1, GenericRecord o2) {
return o1.get("_row_key").toString().compareTo(o2.get("_row_key").toString());
}
});
assertEquals(expectedKey1sand0s, recordsByPrefix);
}