lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-4097] add table info to jobStatus (#5529)

Browse Source 
Co-authored-by: wqwl611 <wqwl611@gmail.com>
This commit is contained in:
wqwl611
2022-05-14 09:01:15 +08:00
committed by GitHub
parent 5c4813f101
commit 52e63b39d6
27 changed files with 41 additions and 41 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
hudi-utilities/src/main/java/org/apache/hudi/utilities/HDFSParquetImporter.java
View File

@@ -174,7 +174,7 @@ public class HDFSParquetImporter implements Serializable {
ParquetInputFormat.setReadSupportClass(job, (AvroReadSupport.class));
HoodieEngineContext context = new HoodieSparkEngineContext(jsc);
context.setJobStatus(this.getClass().getSimpleName(), "Build records for import");
context.setJobStatus(this.getClass().getSimpleName(), "Build records for import: " + cfg.tableName);
return jsc.newAPIHadoopFile(cfg.srcPath, ParquetInputFormat.class, Void.class, GenericRecord.class,
job.getConfiguration())
// To reduce large number of tasks.

2
hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieSnapshotCopier.java
View File

@@ -107,7 +107,7 @@ public class HoodieSnapshotCopier implements Serializable {
fs.delete(new Path(outputDir), true);
}
context.setJobStatus(this.getClass().getSimpleName(), "Creating a snapshot");
context.setJobStatus(this.getClass().getSimpleName(), "Creating a snapshot: " + baseDir);
List<Tuple2<String, String>> filesToCopy = context.flatMap(partitions, partition -> {
// Only take latest version files <= latestCommit.

2
hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieSnapshotExporter.java
View File

@@ -177,7 +177,7 @@ public class HoodieSnapshotExporter {
: ReflectionUtils.loadClass(cfg.outputPartitioner);
HoodieEngineContext context = new HoodieSparkEngineContext(jsc);
context.setJobStatus(this.getClass().getSimpleName(), "Exporting as non-HUDI dataset");
context.setJobStatus(this.getClass().getSimpleName(), "Exporting as non-HUDI dataset: " + cfg.targetOutputPath);
final BaseFileOnlyView fsView = getBaseFileOnlyView(jsc, cfg);
Iterator<String> exportingFilePaths = jsc
.parallelize(partitions, partitions.size())