lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI -409] Match header and footer block length to improve corrupted block detection (#1332)

Browse Source
This commit is contained in:
Ramachandran M S
2020-03-03 13:26:54 -08:00
committed by GitHub
parent 8306205d7a
commit 9d46ce380a
3 changed files with 36 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

21
hudi-common/src/test/java/org/apache/hudi/common/table/log/TestHoodieLogFormat.java
View File

@@ -491,16 +491,27 @@ public class TestHoodieLogFormat extends HoodieCommonTestHarness {
// create a block with
outputStream.write(HoodieLogFormat.MAGIC);
// Write out a length that does not confirm with the content
outputStream.writeLong(1000);
outputStream.writeLong(474);
outputStream.writeInt(HoodieLogBlockType.AVRO_DATA_BLOCK.ordinal());
outputStream.writeInt(HoodieLogFormat.CURRENT_VERSION);
// Write out a length that does not confirm with the content
outputStream.writeLong(500);
// Write out some bytes
outputStream.writeLong(400);
// Write out incomplete content
outputStream.write("something-random".getBytes());
outputStream.flush();
outputStream.close();
// Append a proper block that is of the missing length of the corrupted block
writer =
HoodieLogFormat.newWriterBuilder().onParentPath(partitionPath).withFileExtension(HoodieLogFile.DELTA_EXTENSION)
.withFileId("test-fileid1").overBaseCommit("100").withFs(fs).build();
records = SchemaTestUtil.generateTestRecords(0, 10);
header.put(HoodieLogBlock.HeaderMetadataType.SCHEMA, getSimpleSchema().toString());
dataBlock = new HoodieAvroDataBlock(records, header);
writer = writer.appendBlock(dataBlock);
writer.close();
// First round of reads - we should be able to read the first block and then EOF
Reader reader = HoodieLogFormat.newReader(fs, writer.getLogFile(), SchemaTestUtil.getSimpleSchema());
assertTrue("First block should be available", reader.hasNext());
@@ -508,6 +519,8 @@ public class TestHoodieLogFormat extends HoodieCommonTestHarness {
assertTrue("We should have corrupted block next", reader.hasNext());
HoodieLogBlock block = reader.next();
assertEquals("The read block should be a corrupt block", HoodieLogBlockType.CORRUPT_BLOCK, block.getBlockType());
assertTrue("Third block should be available", reader.hasNext());
reader.next();
assertFalse("There should be no more block left", reader.hasNext());
reader.close();
@@ -543,6 +556,8 @@ public class TestHoodieLogFormat extends HoodieCommonTestHarness {
reader.next();
assertTrue("We should get the 1st corrupted block next", reader.hasNext());
reader.next();
assertTrue("Third block should be available", reader.hasNext());
reader.next();
assertTrue("We should get the 2nd corrupted block next", reader.hasNext());
block = reader.next();
assertEquals("The read block should be a corrupt block", HoodieLogBlockType.CORRUPT_BLOCK, block.getBlockType());