lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-3170] Do not preserve filename when preserveCommitMetadata enabled (#4512)

Browse Source
This commit is contained in:
Sagar Sumit
2022-01-05 18:39:58 +05:30
committed by GitHub
parent 0e297c0c4c
commit 75133f9942
2 changed files with 5 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieCreateHandle.java
View File

@@ -140,6 +140,8 @@ public class HoodieCreateHandle<T extends HoodieRecordPayload, I, K, O> extends
// Convert GenericRecord to GenericRecord with hoodie commit metadata in schema // Convert GenericRecord to GenericRecord with hoodie commit metadata in schema
IndexedRecord recordWithMetadataInSchema = rewriteRecord((GenericRecord) avroRecord.get()); IndexedRecord recordWithMetadataInSchema = rewriteRecord((GenericRecord) avroRecord.get());
if (preserveHoodieMetadata) { if (preserveHoodieMetadata) {
// do not preserve FILENAME_METADATA_FIELD
recordWithMetadataInSchema.put(HoodieRecord.HOODIE_META_COLUMNS_NAME_TO_POS.get(HoodieRecord.FILENAME_METADATA_FIELD), path.getName());
fileWriter.writeAvro(record.getRecordKey(), recordWithMetadataInSchema); fileWriter.writeAvro(record.getRecordKey(), recordWithMetadataInSchema);
} else { } else {
fileWriter.writeAvroWithMetadata(recordWithMetadataInSchema, record); fileWriter.writeAvroWithMetadata(recordWithMetadataInSchema, record);

3
hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieClientOnCopyOnWriteStorage.java
View File

@@ -1836,9 +1836,12 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
Map<String, List<GenericRecord>> recordsByCommitTime = records.stream() Map<String, List<GenericRecord>> recordsByCommitTime = records.stream()
.collect(Collectors.groupingBy(r -> r.get(HoodieRecord.COMMIT_TIME_METADATA_FIELD).toString())); .collect(Collectors.groupingBy(r -> r.get(HoodieRecord.COMMIT_TIME_METADATA_FIELD).toString()));
assertTrue(commitTimes.containsAll(recordsByCommitTime.keySet())); assertTrue(commitTimes.containsAll(recordsByCommitTime.keySet()));
Set<String> expectedFileIds = allStatus.stream().map(WriteStatus::getFileId).collect(Collectors.toSet());
for (GenericRecord record : records) { for (GenericRecord record : records) {
String recordKey = record.get(HoodieRecord.RECORD_KEY_METADATA_FIELD).toString(); String recordKey = record.get(HoodieRecord.RECORD_KEY_METADATA_FIELD).toString();
assertTrue(expectedKeys.contains(recordKey)); assertTrue(expectedKeys.contains(recordKey));
String fileName = record.get(HoodieRecord.FILENAME_METADATA_FIELD).toString();
assertTrue(expectedFileIds.contains(FSUtils.getFileId(fileName)));
} }
} }