lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-707] Add unit test for StatsCommand (#1645)

Browse Source
This commit is contained in:
hongdd
2020-05-21 18:28:04 +08:00
committed by GitHub
parent 6a0aa9a645
commit 802d16c8c9
7 changed files with 260 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
hudi-cli/src/main/java/org/apache/hudi/cli/HoodieTableHeaderFields.java
View File

@@ -96,4 +96,21 @@ public class HoodieTableHeaderFields {
public static final String HEADER_TOTAL_PARTITIONS = "Total Partitions";
public static final String HEADER_DELETED_FILE = "Deleted File";
public static final String HEADER_SUCCEEDED = "Succeeded";
/**
* Fields of Stats.
*/
public static final String HEADER_COMMIT_TIME = "CommitTime";
public static final String HEADER_TOTAL_UPSERTED = "Total Upserted";
public static final String HEADER_TOTAL_WRITTEN = "Total Written";
public static final String HEADER_WRITE_AMPLIFICATION_FACTOR = "Write Amplification Factor";
public static final String HEADER_HISTOGRAM_MIN = "Min";
public static final String HEADER_HISTOGRAM_10TH = "10th";
public static final String HEADER_HISTOGRAM_50TH = "50th";
public static final String HEADER_HISTOGRAM_AVG = "avg";
public static final String HEADER_HISTOGRAM_95TH = "95th";
public static final String HEADER_HISTOGRAM_MAX = "Max";
public static final String HEADER_HISTOGRAM_NUM_FILES = "NumFiles";
public static final String HEADER_HISTOGRAM_STD_DEV = "StdDev";
}

31
hudi-cli/src/main/java/org/apache/hudi/cli/commands/StatsCommand.java
View File

@@ -20,6 +20,7 @@ package org.apache.hudi.cli.commands;
import org.apache.hudi.cli.HoodieCLI;
import org.apache.hudi.cli.HoodiePrintHelper;
import org.apache.hudi.cli.HoodieTableHeaderFields;
import org.apache.hudi.cli.TableHeader;
import org.apache.hudi.common.fs.FSUtils;
import org.apache.hudi.common.model.HoodieCommitMetadata;
@@ -54,7 +55,7 @@ import java.util.stream.Collectors;
@Component
public class StatsCommand implements CommandMarker {
private static final int MAX_FILES = 1000000;
public static final int MAX_FILES = 1000000;
@CliCommand(value = "stats wa", help = "Write Amplification. Ratio of how many records were upserted to how many "
+ "records were actually written")
@@ -92,12 +93,14 @@ public class StatsCommand implements CommandMarker {
}
rows.add(new Comparable[] {"Total", totalRecordsUpserted, totalRecordsWritten, waf});
TableHeader header = new TableHeader().addTableHeaderField("CommitTime").addTableHeaderField("Total Upserted")
.addTableHeaderField("Total Written").addTableHeaderField("Write Amplification Factor");
TableHeader header = new TableHeader().addTableHeaderField(HoodieTableHeaderFields.HEADER_COMMIT_TIME)
.addTableHeaderField(HoodieTableHeaderFields.HEADER_TOTAL_UPSERTED)
.addTableHeaderField(HoodieTableHeaderFields.HEADER_TOTAL_WRITTEN)
.addTableHeaderField(HoodieTableHeaderFields.HEADER_WRITE_AMPLIFICATION_FACTOR);
return HoodiePrintHelper.print(header, new HashMap<>(), sortByField, descending, limit, headerOnly, rows);
}
private Comparable[] printFileSizeHistogram(String instantTime, Snapshot s) {
public Comparable[] printFileSizeHistogram(String instantTime, Snapshot s) {
return new Comparable[] {instantTime, s.getMin(), s.getValue(0.1), s.getMedian(), s.getMean(), s.get95thPercentile(),
s.getMax(), s.size(), s.getStdDev()};
}
@@ -138,6 +141,20 @@ public class StatsCommand implements CommandMarker {
Snapshot s = globalHistogram.getSnapshot();
rows.add(printFileSizeHistogram("ALL", s));
TableHeader header = new TableHeader()
.addTableHeaderField(HoodieTableHeaderFields.HEADER_COMMIT_TIME)
.addTableHeaderField(HoodieTableHeaderFields.HEADER_HISTOGRAM_MIN)
.addTableHeaderField(HoodieTableHeaderFields.HEADER_HISTOGRAM_10TH)
.addTableHeaderField(HoodieTableHeaderFields.HEADER_HISTOGRAM_50TH)
.addTableHeaderField(HoodieTableHeaderFields.HEADER_HISTOGRAM_AVG)
.addTableHeaderField(HoodieTableHeaderFields.HEADER_HISTOGRAM_95TH)
.addTableHeaderField(HoodieTableHeaderFields.HEADER_HISTOGRAM_MAX)
.addTableHeaderField(HoodieTableHeaderFields.HEADER_HISTOGRAM_NUM_FILES)
.addTableHeaderField(HoodieTableHeaderFields.HEADER_HISTOGRAM_STD_DEV);
return HoodiePrintHelper.print(header, getFieldNameToConverterMap(), sortByField, descending, limit, headerOnly, rows);
}
public Map<String, Function<Object, String>> getFieldNameToConverterMap() {
Function<Object, String> converterFunction =
entry -> NumericUtils.humanReadableByteCount((Double.parseDouble(entry.toString())));
Map<String, Function<Object, String>> fieldNameToConverterMap = new HashMap<>();
@@ -148,10 +165,6 @@ public class StatsCommand implements CommandMarker {
fieldNameToConverterMap.put("95th", converterFunction);
fieldNameToConverterMap.put("Max", converterFunction);
fieldNameToConverterMap.put("StdDev", converterFunction);
TableHeader header = new TableHeader().addTableHeaderField("CommitTime").addTableHeaderField("Min")
.addTableHeaderField("10th").addTableHeaderField("50th").addTableHeaderField("avg").addTableHeaderField("95th")
.addTableHeaderField("Max").addTableHeaderField("NumFiles").addTableHeaderField("StdDev");
return HoodiePrintHelper.print(header, fieldNameToConverterMap, sortByField, descending, limit, headerOnly, rows);
return fieldNameToConverterMap;
}
}