lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-4192] HoodieHFileReader scan top cells after bottom cells throw NullPointerException (#5755)

Browse Source 
SeekTo top cells avoid NullPointerException
This commit is contained in:
marchpure
2022-06-06 12:07:26 +08:00
committed by GitHub
parent 5d18b80343
commit 73b0be3c96
2 changed files with 38 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

32
hudi-client/hudi-client-common/src/test/java/org/apache/hudi/io/storage/TestHoodieHFileReaderWriter.java
View File

@@ -294,6 +294,38 @@ public class TestHoodieHFileReaderWriter extends TestHoodieReaderWriterBase {
StreamSupport.stream(Spliterators.spliteratorUnknownSize(iterator, Spliterator.ORDERED), false) StreamSupport.stream(Spliterators.spliteratorUnknownSize(iterator, Spliterator.ORDERED), false)
.collect(Collectors.toList()); .collect(Collectors.toList());
assertEquals(Collections.emptyList(), recordsByPrefix); assertEquals(Collections.emptyList(), recordsByPrefix);
// filter for "key50" and "key1" : entries from key50 and 'key10 to key19' should be matched.
List<GenericRecord> expectedKey50and1s = allRecords.stream().filter(entry -> (entry.get("_row_key").toString()).contains("key1")
|| (entry.get("_row_key").toString()).contains("key50")).collect(Collectors.toList());
iterator =
hfileReader.getRecordsByKeyPrefixIterator(Arrays.asList("key50", "key1"), avroSchema);
recordsByPrefix =
StreamSupport.stream(Spliterators.spliteratorUnknownSize(iterator, Spliterator.ORDERED), false)
.collect(Collectors.toList());
assertEquals(expectedKey50and1s, recordsByPrefix);
// filter for "key50" and "key0" : entries from key50 and 'key00 to key09' should be matched.
List<GenericRecord> expectedKey50and0s = allRecords.stream().filter(entry -> (entry.get("_row_key").toString()).contains("key0")
|| (entry.get("_row_key").toString()).contains("key50")).collect(Collectors.toList());
iterator =
hfileReader.getRecordsByKeyPrefixIterator(Arrays.asList("key50", "key0"), avroSchema);
recordsByPrefix =
StreamSupport.stream(Spliterators.spliteratorUnknownSize(iterator, Spliterator.ORDERED), false)
.collect(Collectors.toList());
assertEquals(expectedKey50and0s, recordsByPrefix);
// filter for "key1" and "key0" : entries from 'key10 to key19' and 'key00 to key09' should be matched.
List<GenericRecord> expectedKey1sand0s = expectedKey1s;
expectedKey1sand0s.addAll(allRecords.stream()
.filter(entry -> (entry.get("_row_key").toString()).contains("key0"))
.collect(Collectors.toList()));
iterator =
hfileReader.getRecordsByKeyPrefixIterator(Arrays.asList("key1", "key0"), avroSchema);
recordsByPrefix =
StreamSupport.stream(Spliterators.spliteratorUnknownSize(iterator, Spliterator.ORDERED), false)
.collect(Collectors.toList());
assertEquals(expectedKey1sand0s, recordsByPrefix);
} }
@ParameterizedTest @ParameterizedTest

6
hudi-common/src/main/java/org/apache/hudi/io/storage/HoodieHFileReader.java
View File

@@ -258,6 +258,12 @@ public class HoodieHFileReader<R extends IndexedRecord> implements HoodieFileRea
if (!scanner.next()) { if (!scanner.next()) {
return Collections.emptyIterator(); return Collections.emptyIterator();
} }
} else if (val == -1) {
// If scanner is aleady on the top of hfile. avoid trigger seekTo again.
Option<Cell> headerCell = Option.fromJavaOptional(scanner.getReader().getFirstKey());
if (headerCell.isPresent() && !headerCell.get().equals(scanner.getCell())) {
scanner.seekTo();
}
} }
class KeyPrefixIterator implements Iterator<GenericRecord> { class KeyPrefixIterator implements Iterator<GenericRecord> {