1
0

[HUDI-2660] Delete the view storage properties first before creation (#3899)

This commit is contained in:
Danny Chan
2021-11-03 14:30:20 +08:00
committed by GitHub
parent 5517d292f9
commit 7fc7e9b2bc
3 changed files with 8 additions and 1 deletions

View File

@@ -231,7 +231,7 @@ public class BootstrapOperator<I, O extends HoodieRecord>
return hoodieRecord;
}
private static boolean shouldLoadFile(String fileId,
protected boolean shouldLoadFile(String fileId,
int maxParallelism,
int parallelism,
int taskID) {

View File

@@ -75,4 +75,10 @@ public class BatchBootstrapOperator<I, O extends HoodieRecord>
// send the trigger record
output.collect((StreamRecord<O>) element);
}
@Override
protected boolean shouldLoadFile(String fileId, int maxParallelism, int parallelism, int taskID) {
// load all the file groups in the partition
return true;
}
}

View File

@@ -51,6 +51,7 @@ public class ViewStorageProperties {
FileSystemViewStorageConfig config) throws IOException {
Path propertyPath = getPropertiesFilePath(basePath);
FileSystem fs = FSUtils.getFs(basePath, StreamerUtil.getHadoopConf());
fs.delete(propertyPath, false);
try (FSDataOutputStream outputStream = fs.create(propertyPath)) {
config.getProps().store(outputStream,
"Filesystem view storage properties saved on " + new Date(System.currentTimeMillis()));