lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-3709] Fixing ParquetWriter impls not respecting Parquet Max File Size limit (#5129)

Browse Source
This commit is contained in:
Alexey Kudinkin
2022-03-26 14:51:36 -07:00
committed by GitHub
parent 57b4f39c31
commit 189d5297b8
7 changed files with 4 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/row/HoodieInternalRowParquetWriter.java
View File

@@ -56,7 +56,7 @@ public class HoodieInternalRowParquetWriter extends ParquetWriter<InternalRow>
@Override
public boolean canWrite() {
return fs.getBytesWritten(file) < maxFileSize;
return getDataSize() < maxFileSize;
}
@Override

2
hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/action/commit/TestCopyOnWriteActionExecutor.java
View File

@@ -437,7 +437,7 @@ public class TestCopyOnWriteActionExecutor extends HoodieClientTestBase {
counts++;
}
}
assertEquals(3, counts, "If the number of records are more than 1150, then there should be a new file");
assertEquals(5, counts, "If the number of records are more than 1150, then there should be a new file");
}
@Test