lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-3283] Bootstrap support overwrite existing table (#4647)

Browse Source
This commit is contained in:
wangxianghu
2022-01-20 14:42:52 +04:00
committed by GitHub
parent 31b57a256f
commit b7a79aa943
2 changed files with 18 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

23
hudi-utilities/src/main/java/org/apache/hudi/utilities/deltastreamer/BootstrapExecutor.java
View File

@@ -54,12 +54,12 @@ import static org.apache.hudi.common.table.HoodieTableConfig.ARCHIVELOG_FOLDER;
/** /**
* Performs bootstrap from a non-hudi source. * Performs bootstrap from a non-hudi source.
*/ */
public class BootstrapExecutor implements Serializable { public class BootstrapExecutor implements Serializable {
private static final Logger LOG = LogManager.getLogger(BootstrapExecutor.class); private static final Logger LOG = LogManager.getLogger(BootstrapExecutor.class);
/** /**
* Config. * Config.
*/ */
private final HoodieDeltaStreamer.Config cfg; private final HoodieDeltaStreamer.Config cfg;
@@ -97,9 +97,10 @@ public class BootstrapExecutor implements Serializable {
/** /**
* Bootstrap Executor. * Bootstrap Executor.
* @param cfg DeltaStreamer Config *
* @param jssc Java Spark Context * @param cfg DeltaStreamer Config
* @param fs File System * @param jssc Java Spark Context
* @param fs File System
* @param properties Bootstrap Writer Properties * @param properties Bootstrap Writer Properties
* @throws IOException * @throws IOException
*/ */
@@ -168,9 +169,15 @@ public class BootstrapExecutor implements Serializable {
} }
private void initializeTable() throws IOException { private void initializeTable() throws IOException {
if (fs.exists(new Path(cfg.targetBasePath))) { Path basePath = new Path(cfg.targetBasePath);
throw new HoodieException("target base path already exists at " + cfg.targetBasePath if (fs.exists(basePath)) {
+ ". Cannot bootstrap data on top of an existing table"); if (cfg.bootstrapOverwrite) {
LOG.warn("Target base path already exists, overwrite it");
fs.delete(basePath, true);
} else {
throw new HoodieException("target base path already exists at " + cfg.targetBasePath
+ ". Cannot bootstrap data on top of an existing table");
}
} }
HoodieTableMetaClient.withPropertyBuilder() HoodieTableMetaClient.withPropertyBuilder()
.setTableType(cfg.tableType) .setTableType(cfg.tableType)

3
hudi-utilities/src/main/java/org/apache/hudi/utilities/deltastreamer/HoodieDeltaStreamer.java
View File

@@ -363,6 +363,9 @@ public class HoodieDeltaStreamer implements Serializable {
@Parameter(names = {"--run-bootstrap"}, description = "Run bootstrap if bootstrap index is not found") @Parameter(names = {"--run-bootstrap"}, description = "Run bootstrap if bootstrap index is not found")
public Boolean runBootstrap = false; public Boolean runBootstrap = false;
@Parameter(names = {"--bootstrap-overwrite"}, description = "Overwrite existing target table, default false")
public Boolean bootstrapOverwrite = false;
@Parameter(names = {"--bootstrap-index-class"}, description = "subclass of BootstrapIndex") @Parameter(names = {"--bootstrap-index-class"}, description = "subclass of BootstrapIndex")
public String bootstrapIndexClass = HFileBootstrapIndex.class.getName(); public String bootstrapIndexClass = HFileBootstrapIndex.class.getName();