lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

HUDI-131 Zero FIle Listing in Compactor run

Browse Source
This commit is contained in:
Balaji Varadarajan
2019-05-24 16:51:09 -07:00
committed by vinoth chandar
parent 4074c5eb23
commit 99b0c72aa6
2 changed files with 7 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
hoodie-client/src/main/java/com/uber/hoodie/io/compact/HoodieRealtimeTableCompactor.java
View File

@@ -126,14 +126,13 @@ public class HoodieRealtimeTableCompactor implements HoodieCompactor {
return Lists.<WriteStatus>newArrayList(); return Lists.<WriteStatus>newArrayList();
} }
Option<HoodieDataFile> oldDataFileOpt = hoodieCopyOnWriteTable.getROFileSystemView() Option<HoodieDataFile> oldDataFileOpt = operation.getBaseFile();
.getDataFileOn(operation.getPartitionPath(), operation.getBaseInstantTime(), operation.getFileId());
// Compacting is very similar to applying updates to existing file // Compacting is very similar to applying updates to existing file
Iterator<List<WriteStatus>> result; Iterator<List<WriteStatus>> result;
// If the dataFile is present, there is a base parquet file present, perform updates else perform inserts into a // If the dataFile is present, there is a base parquet file present, perform updates else perform inserts into a
// new base parquet file. // new base parquet file.
if (operation.getDataFilePath().isPresent()) { if (oldDataFileOpt.isPresent()) {
result = hoodieCopyOnWriteTable result = hoodieCopyOnWriteTable
.handleUpdate(commitTime, operation.getFileId(), scanner.getRecords(), oldDataFileOpt.get()); .handleUpdate(commitTime, operation.getFileId(), scanner.getRecords(), oldDataFileOpt.get());
} else { } else {

5
hoodie-common/src/main/java/com/uber/hoodie/common/model/CompactionOperation.java
View File

@@ -110,6 +110,11 @@ public class CompactionOperation implements Serializable {
return id; return id;
} }
public Option<HoodieDataFile> getBaseFile() {
//TODO: HUDI-130 - Paths return in compaction plan needs to be relative to base-path
return dataFilePath.map(df -> new HoodieDataFile(df));
}
/** /**
* Convert Avro generated Compaction operation to POJO for Spark RDD operation * Convert Avro generated Compaction operation to POJO for Spark RDD operation
* @param operation Hoodie Compaction Operation * @param operation Hoodie Compaction Operation