lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-2660] Delete the view storage properties first before creation (#3899)

Browse Source
This commit is contained in:
Danny Chan
2021-11-03 14:30:20 +08:00
committed by GitHub
parent 5517d292f9
commit 7fc7e9b2bc
3 changed files with 8 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
hudi-flink/src/main/java/org/apache/hudi/sink/bootstrap/BootstrapOperator.java
View File

@@ -231,7 +231,7 @@ public class BootstrapOperator<I, O extends HoodieRecord>
return hoodieRecord; return hoodieRecord;
} }
private static boolean shouldLoadFile(String fileId, protected boolean shouldLoadFile(String fileId,
int maxParallelism, int maxParallelism,
int parallelism, int parallelism,
int taskID) { int taskID) {

6
hudi-flink/src/main/java/org/apache/hudi/sink/bootstrap/batch/BatchBootstrapOperator.java
View File

@@ -75,4 +75,10 @@ public class BatchBootstrapOperator<I, O extends HoodieRecord>
// send the trigger record // send the trigger record
output.collect((StreamRecord<O>) element); output.collect((StreamRecord<O>) element);
} }
@Override
protected boolean shouldLoadFile(String fileId, int maxParallelism, int parallelism, int taskID) {
// load all the file groups in the partition
return true;
}
} }

1
hudi-flink/src/main/java/org/apache/hudi/util/ViewStorageProperties.java
View File

@@ -51,6 +51,7 @@ public class ViewStorageProperties {
FileSystemViewStorageConfig config) throws IOException { FileSystemViewStorageConfig config) throws IOException {
Path propertyPath = getPropertiesFilePath(basePath); Path propertyPath = getPropertiesFilePath(basePath);
FileSystem fs = FSUtils.getFs(basePath, StreamerUtil.getHadoopConf()); FileSystem fs = FSUtils.getFs(basePath, StreamerUtil.getHadoopConf());
fs.delete(propertyPath, false);
try (FSDataOutputStream outputStream = fs.create(propertyPath)) { try (FSDataOutputStream outputStream = fs.create(propertyPath)) {
config.getProps().store(outputStream, config.getProps().store(outputStream,
"Filesystem view storage properties saved on " + new Date(System.currentTimeMillis())); "Filesystem view storage properties saved on " + new Date(System.currentTimeMillis()));