lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

Adding verbose output for metadata validate files command (#4166)

Browse Source
This commit is contained in:
Sivabalan Narayanan
2021-12-10 12:38:38 -05:00
committed by GitHub
parent 3ad9b121f1
commit 3ce0526924
1 changed files with 16 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

24
hudi-cli/src/main/java/org/apache/hudi/cli/commands/MetadataCommand.java
View File

@@ -62,7 +62,7 @@ import java.util.Set;
* Example: * Example:
* The default spark.master conf is set to yarn. If you are running on a local deployment, * The default spark.master conf is set to yarn. If you are running on a local deployment,
* we can set the spark master to local using set conf command. * we can set the spark master to local using set conf command.
* > set --conf SPARK_MASTER=local * > set --conf SPARK_MASTER=local[2]
* <p> * <p>
* Connect to the table * Connect to the table
* > connect --path {path to hudi table} * > connect --path {path to hudi table}
@@ -297,12 +297,20 @@ public class MetadataCommand implements CommandMarker {
row[0] = partition; row[0] = partition;
FileStatus fsFileStatus = fileStatusMap.get(file); FileStatus fsFileStatus = fileStatusMap.get(file);
FileStatus metaFileStatus = metadataFileStatusMap.get(file); FileStatus metaFileStatus = metadataFileStatusMap.get(file);
boolean doesFsFileExists = fsFileStatus != null;
boolean doesMetadataFileExists = metaFileStatus != null;
long fsFileLength = doesFsFileExists ? fsFileStatus.getLen() : 0;
long metadataFileLength = doesMetadataFileExists ? metaFileStatus.getLen() : 0;
row[1] = file; row[1] = file;
row[2] = fsFileStatus != null; row[2] = doesFsFileExists;
row[3] = metaFileStatus != null; row[3] = doesMetadataFileExists;
row[4] = (fsFileStatus != null) ? fsFileStatus.getLen() : 0; row[4] = fsFileLength;
row[5] = (metaFileStatus != null) ? metaFileStatus.getLen() : 0; row[5] = metadataFileLength;
rows.add(row); if (verbose) { // if verbose print all files
rows.add(row);
} else if ((doesFsFileExists != doesMetadataFileExists) || (fsFileLength != metadataFileLength)) { // if non verbose, print only non matching files
rows.add(row);
}
} }
if (metadataStatuses.length != fsStatuses.length) { if (metadataStatuses.length != fsStatuses.length) {
@@ -337,8 +345,8 @@ public class MetadataCommand implements CommandMarker {
} }
TableHeader header = new TableHeader().addTableHeaderField("Partition") TableHeader header = new TableHeader().addTableHeaderField("Partition")
.addTableHeaderField("File Name") .addTableHeaderField("File Name")
.addTableHeaderField(" IsPresent in FS ") .addTableHeaderField(" Is Present in FS ")
.addTableHeaderField(" IsPresent in Metadata") .addTableHeaderField(" Is Present in Metadata")
.addTableHeaderField(" FS size") .addTableHeaderField(" FS size")
.addTableHeaderField(" Metadata size"); .addTableHeaderField(" Metadata size");
return HoodiePrintHelper.print(header, new HashMap<>(), "", false, Integer.MAX_VALUE, false, rows); return HoodiePrintHelper.print(header, new HashMap<>(), "", false, Integer.MAX_VALUE, false, rows);