lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity
Files
c319ee9cea78544406e30dd36cc603fd0a0283db
hudi/scripts/grafana-dashboard.json
Prashant Wason 7173d1338a [HUDI-2124] A Grafana dashboard for HUDI. (#3216)
2021-07-02 18:48:37 -07:00

8336 lines
237 KiB
JSON
Raw Blame History

{
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": "-- Grafana --",
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"type": "dashboard"
}
]
},
"description": "HUDI Dashboard which has been contributed to the OSS project",
"editable": true,
"gnetId": null,
"graphTooltip": 1,
"id": 124311,
"iteration": 1625260824034,
"links": [],
"panels": [
{
"collapsed": true,
"datasource": null,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 0
},
"id": 83,
"panels": [
{
"cacheTimeout": null,
"datasource": "M3QL",
"fieldConfig": {
"defaults": {
"custom": {},
"mappings": [
{
"id": 0,
"op": "=",
"text": "N/A",
"type": 1,
"value": "null"
}
],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
},
"unit": "short"
},
"overrides": []
},
"gridPos": {
"h": 6,
"w": 3,
"x": 0,
"y": 1
},
"id": 81,
"interval": null,
"links": [],
"maxDataPoints": 100,
"options": {
"colorMode": "value",
"graphMode": "none",
"justifyMode": "auto",
"orientation": "horizontal",
"reduceOptions": {
"calcs": [
"mean"
],
"fields": "",
"values": false
},
"text": {},
"textMode": "auto"
},
"pluginVersion": "7.4.2",
"targets": [
{
"refId": "A",
"target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.commit.duration | removeEmpty | sum name | count\n"
}
],
"title": "Dataset Count",
"type": "stat"
},
{
"cacheTimeout": null,
"datasource": "M3QL",
"fieldConfig": {
"defaults": {
"custom": {},
"mappings": [
{
"id": 0,
"op": "=",
"text": "N/A",
"type": 1,
"value": "null"
}
],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
},
"unit": "short"
},
"overrides": []
},
"gridPos": {
"h": 6,
"w": 3,
"x": 3,
"y": 1
},
"id": 215,
"interval": null,
"links": [],
"maxDataPoints": 100,
"options": {
"colorMode": "value",
"graphMode": "none",
"justifyMode": "auto",
"orientation": "horizontal",
"reduceOptions": {
"calcs": [
"mean"
],
"fields": "",
"values": false
},
"text": {},
"textMode": "auto"
},
"pluginVersion": "7.4.2",
"targets": [
{
"refId": "A",
"target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.HoodieMetadata.basefile.count | removeEmpty | sumSeries name | count\n"
}
],
"title": "Datasets with Metadata Enabled",
"type": "stat"
},
{
"cacheTimeout": null,
"datasource": "M3QL",
"fieldConfig": {
"defaults": {
"custom": {},
"mappings": [
{
"id": 0,
"op": "=",
"text": "N/A",
"type": 1,
"value": "null"
}
],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
},
"unit": "short"
},
"overrides": []
},
"gridPos": {
"h": 6,
"w": 3,
"x": 6,
"y": 1
},
"id": 147,
"interval": null,
"links": [],
"maxDataPoints": 100,
"options": {
"colorMode": "value",
"graphMode": "none",
"justifyMode": "auto",
"orientation": "horizontal",
"reduceOptions": {
"calcs": [
"mean"
],
"fields": "",
"values": false
},
"text": {},
"textMode": "auto"
},
"pluginVersion": "7.4.2",
"targets": [
{
"refId": "A",
"target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.commit.totalInsertRecordsWritten | sum | moving 1d sum\n"
}
],
"title": "Records Inserted / Day",
"type": "stat"
},
{
"cacheTimeout": null,
"datasource": "M3QL",
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"custom": {},
"mappings": [
{
"id": 0,
"op": "=",
"text": "N/A",
"type": 1,
"value": "null"
}
],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
},
"unit": "short"
},
"overrides": []
},
"gridPos": {
"h": 6,
"w": 3,
"x": 9,
"y": 1
},
"id": 148,
"interval": null,
"links": [],
"maxDataPoints": 100,
"options": {
"colorMode": "value",
"graphMode": "none",
"justifyMode": "auto",
"orientation": "horizontal",
"reduceOptions": {
"calcs": [
"mean"
],
"fields": "",
"values": false
},
"text": {},
"textMode": "auto"
},
"pluginVersion": "7.4.2",
"targets": [
{
"refId": "A",
"target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.commit.totalUpdateRecordsWritten | sum | moving 1d sum"
}
],
"title": "Records Upserted / Day",
"type": "stat"
},
{
"cacheTimeout": null,
"datasource": "M3QL",
"fieldConfig": {
"defaults": {
"custom": {},
"mappings": [
{
"id": 0,
"op": "=",
"text": "N/A",
"type": 1,
"value": "null"
}
],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
},
"unit": "short"
},
"overrides": []
},
"gridPos": {
"h": 6,
"w": 3,
"x": 12,
"y": 1
},
"id": 150,
"interval": null,
"links": [],
"maxDataPoints": 100,
"options": {
"colorMode": "value",
"graphMode": "none",
"justifyMode": "center",
"orientation": "horizontal",
"reduceOptions": {
"calcs": [
"mean"
],
"fields": "",
"values": false
},
"text": {},
"textMode": "value"
},
"pluginVersion": "7.4.2",
"targets": [
{
"refId": "A",
"target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.commit.totalRecordsWritten | sum | moving 1d sum"
}
],
"title": "Records Written / Day",
"type": "stat"
},
{
"cacheTimeout": null,
"datasource": null,
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"custom": {},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": []
},
"unit": "short"
},
"overrides": []
},
"gridPos": {
"h": 6,
"w": 3,
"x": 15,
"y": 1
},
"id": 237,
"interval": null,
"links": [],
"maxDataPoints": 100,
"options": {
"colorMode": "value",
"graphMode": "none",
"justifyMode": "center",
"orientation": "horizontal",
"reduceOptions": {
"calcs": [
"mean"
],
"fields": "",
"values": false
},
"text": {},
"textMode": "value"
},
"pluginVersion": "7.4.2",
"targets": [
{
"refId": "A",
"target": "countSeries(groupByNode(stats.$dc.gauges.hoodie.$Environment.$Dataset.$Tablefilter.userName.*, -1, 'sum'))",
"textEditor": false
}
],
"title": "Active Users",
"type": "stat"
},
{
"aliasColors": {},
"breakPoint": "50%",
"cacheTimeout": null,
"combine": {
"label": "Others",
"threshold": 0
},
"datasource": "M3QL",
"fieldConfig": {
"defaults": {
"custom": {}
},
"overrides": []
},
"fontSize": "80%",
"format": "short",
"gridPos": {
"h": 6,
"w": 4,
"x": 18,
"y": 1
},
"id": 236,
"interval": null,
"legend": {
"percentage": false,
"show": true,
"values": true
},
"legendType": "Right side",
"links": [],
"maxDataPoints": 1,
"nullPointMode": "connected",
"pieType": "donut",
"pluginVersion": "7.4.2",
"strokeWidth": 1,
"targets": [
{
"refId": "A",
"target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.version.0.7.0 | sumSeries name | count | alias \"0.7\""
},
{
"hide": false,
"refId": "B",
"target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.finalize.duration | sumSeries name | count | diff (fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.version.0.7.0 | sumSeries name | count) | alias \"0.5\""
}
],
"title": "Versions in Production",
"type": "grafana-piechart-panel",
"valueName": "current"
},
{
"cacheTimeout": null,
"datasource": "M3QL",
"description": "Total number of Clean operations / day ",
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"custom": {},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": []
},
"unit": "short"
},
"overrides": []
},
"gridPos": {
"h": 6,
"w": 3,
"x": 0,
"y": 7
},
"id": 240,
"interval": null,
"links": [],
"maxDataPoints": 100,
"options": {
"colorMode": "value",
"graphMode": "none",
"justifyMode": "center",
"orientation": "horizontal",
"reduceOptions": {
"calcs": [
"mean"
],
"fields": "",
"values": false
},
"text": {},
"textMode": "value"
},
"pluginVersion": "7.4.2",
"targets": [
{
"refId": "A",
"target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.clean.duration | removeEmpty | isNonNull | sum | moving 1D sum\n"
},
{
"hide": true,
"refId": "B",
"target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.${Tablefilter}.HoodieMetadata.lookup_files.count | removeEmpty | sum | alias \"lookup_files\""
}
],
"title": "Clean Operations per / Day",
"type": "stat"
},
{
"cacheTimeout": null,
"datasource": "M3QL",
"description": "Total number of Commit operations / day ",
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"custom": {},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": []
},
"unit": "short"
},
"overrides": []
},
"gridPos": {
"h": 6,
"w": 3,
"x": 3,
"y": 7
},
"id": 241,
"interval": null,
"links": [],
"maxDataPoints": 100,
"options": {
"colorMode": "value",
"graphMode": "none",
"justifyMode": "center",
"orientation": "horizontal",
"reduceOptions": {
"calcs": [
"mean"
],
"fields": "",
"values": false
},
"text": {},
"textMode": "value"
},
"pluginVersion": "7.4.2",
"targets": [
{
"refId": "A",
"target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.{commit,deltacommit}.duration | removeEmpty | isNonNull | sum | moving 1D sum\n"
},
{
"hide": true,
"refId": "B",
"target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.${Tablefilter}.HoodieMetadata.lookup_files.count | removeEmpty | sum | alias \"lookup_files\""
}
],
"title": "Commit Operations per / Day",
"type": "stat"
}
],
"title": "Summary",
"type": "row"
},
{
"collapsed": true,
"datasource": null,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 1
},
"id": 84,
"panels": [
{
"aliasColors": {
"ONCALL_LINE": "#BF1B00"
},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": null,
"editable": true,
"error": false,
"fieldConfig": {
"defaults": {
"custom": {}
},
"overrides": []
},
"fill": 0,
"fillGradient": 0,
"grid": {},
"gridPos": {
"h": 7,
"w": 24,
"x": 0,
"y": 2
},
"hiddenSeries": false,
"id": 87,
"legend": {
"avg": true,
"current": false,
"max": false,
"min": false,
"show": true,
"total": false,
"values": true
},
"lines": true,
"linewidth": 1,
"links": [],
"nullPointMode": "connected",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "7.4.2",
"pointradius": 1,
"points": true,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"refId": "A",
"target": "aliasByNode(highestMax(stats.$dc.gauges.hoodie.$Environment.$Dataset.$Tablefilter.commit.duration, 10), 1, 6)",
"textEditor": false
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "Commit Duration (Top 10)",
"tooltip": {
"shared": false,
"sort": 0,
"value_type": "cumulative"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:1819",
"format": "ms",
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"$$hashKey": "object:1820",
"format": "short",
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"aliasColors": {
"ONCALL_LINE": "#BF1B00"
},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": null,
"editable": true,
"error": false,
"fieldConfig": {
"defaults": {
"custom": {}
},
"overrides": []
},
"fill": 0,
"fillGradient": 0,
"grid": {},
"gridPos": {
"h": 7,
"w": 24,
"x": 0,
"y": 9
},
"hiddenSeries": false,
"id": 80,
"legend": {
"avg": true,
"current": false,
"max": false,
"min": false,
"show": false,
"total": false,
"values": true
},
"lines": true,
"linewidth": 1,
"links": [],
"nullPointMode": "connected",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "7.4.2",
"pointradius": 1,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"refId": "A",
"target": "aliasByNode(nPercentile(stats.$dc.gauges.hoodie.$Environment.$Dataset.$Tablefilter.commit.duration, 95), 5, 6)",
"textEditor": true
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "Commit Duration (p95)",
"tooltip": {
"shared": false,
"sort": 0,
"value_type": "cumulative"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:1903",
"format": "ms",
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"$$hashKey": "object:1904",
"format": "short",
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
}
],
"title": "Commit Detailed Metrics",
"type": "row"
},
{
"collapsed": true,
"datasource": null,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 2
},
"id": 228,
"panels": [
{
"aliasColors": {
"ONCALL_LINE": "#BF1B00"
},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "M3QL",
"editable": true,
"error": false,
"fieldConfig": {
"defaults": {
"custom": {}
},
"overrides": []
},
"fill": 0,
"fillGradient": 0,
"grid": {},
"gridPos": {
"h": 9,
"w": 12,
"x": 0,
"y": 3
},
"hiddenSeries": false,
"id": 230,
"legend": {
"avg": true,
"current": false,
"max": false,
"min": false,
"show": true,
"total": false,
"values": true
},
"lines": true,
"linewidth": 1,
"links": [],
"nullPointMode": "connected",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "7.4.2",
"pointradius": 1,
"points": true,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"refId": "A",
"target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.clean.duration | sortSeries max desc | head 10\n"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "Clean Duration (Top 10)",
"tooltip": {
"shared": false,
"sort": 0,
"value_type": "cumulative"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:1819",
"format": "ms",
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"$$hashKey": "object:1820",
"format": "short",
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"aliasColors": {
"ONCALL_LINE": "#BF1B00"
},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "M3QL",
"editable": true,
"error": false,
"fieldConfig": {
"defaults": {
"custom": {}
},
"overrides": []
},
"fill": 0,
"fillGradient": 0,
"grid": {},
"gridPos": {
"h": 9,
"w": 12,
"x": 12,
"y": 3
},
"hiddenSeries": false,
"id": 238,
"legend": {
"avg": true,
"current": false,
"max": false,
"min": false,
"show": true,
"total": false,
"values": true
},
"lines": true,
"linewidth": 1,
"links": [],
"nullPointMode": "connected",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "7.4.2",
"pointradius": 1,
"points": true,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"refId": "A",
"target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.clean.numFilesDeleted | sortSeries max desc | head 10\n"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "Clean File Deleted",
"tooltip": {
"shared": false,
"sort": 0,
"value_type": "cumulative"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:1819",
"format": "short",
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"$$hashKey": "object:1820",
"format": "short",
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"aliasColors": {
"ONCALL_LINE": "#BF1B00"
},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "M3QL",
"editable": true,
"error": false,
"fieldConfig": {
"defaults": {
"custom": {}
},
"overrides": []
},
"fill": 0,
"fillGradient": 0,
"grid": {},
"gridPos": {
"h": 9,
"w": 12,
"x": 0,
"y": 12
},
"hiddenSeries": false,
"id": 231,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 1,
"links": [],
"nullPointMode": "connected",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "7.4.2",
"pointradius": 1,
"points": true,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"refId": "A",
"target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.clean.duration | max | alias \"Maximum\"\n"
},
{
"hide": false,
"refId": "B",
"target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.clean.duration | sum | nPercentile 95 | alias \"P95\"\n"
},
{
"hide": false,
"refId": "C",
"target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.clean.duration | sum | nPercentile 99 | alias \"P99\"\n"
},
{
"hide": false,
"refId": "D",
"target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.clean.duration | sum | moving 1D avg | alias \"AVG\"\n"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "Clean Duration (Max, P95, P99)",
"tooltip": {
"shared": false,
"sort": 0,
"value_type": "cumulative"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:1819",
"format": "ms",
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"$$hashKey": "object:1820",
"format": "short",
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
}
],
"title": "Clean Metrics",
"type": "row"
},
{
"collapsed": true,
"datasource": null,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 3
},
"id": 86,
"panels": [
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": null,
"editable": true,
"error": false,
"fieldConfig": {
"defaults": {
"custom": {}
},
"overrides": []
},
"fill": 1,
"fillGradient": 0,
"grid": {},
"gridPos": {
"h": 7,
"w": 24,
"x": 0,
"y": 4
},
"hiddenSeries": false,
"id": 67,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 2,
"links": [],
"nullPointMode": "connected",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "7.4.2",
"pointradius": 2,
"points": true,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"refId": "A",
"target": "aliasByNode(highestMax(stats.$dc.gauges.hoodie.$Environment.$Dataset.$Tablefilter.commit.totalFilesInsert.count, 10), 1, 6)",
"textEditor": false
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "Number of NEW Files Generated (Top 10)",
"tooltip": {
"shared": false,
"sort": 0,
"value_type": "cumulative"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:2009",
"format": "short",
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"$$hashKey": "object:2010",
"format": "short",
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": null,
"editable": true,
"error": false,
"fieldConfig": {
"defaults": {
"custom": {}
},
"overrides": []
},
"fill": 1,
"fillGradient": 0,
"grid": {},
"gridPos": {
"h": 7,
"w": 24,
"x": 0,
"y": 11
},
"hiddenSeries": false,
"id": 73,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 2,
"links": [],
"nullPointMode": "connected",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "7.4.2",
"pointradius": 2,
"points": true,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"refId": "A",
"target": "aliasByNode(highestMax(stats.$dc.gauges.hoodie.$Environment.$Dataset.$Tablefilter.commit.totalFilesUpdate.count, 10), 1, 6)",
"textEditor": true
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "Number of EXISTING files versioned (Top 10)",
"tooltip": {
"shared": false,
"sort": 0,
"value_type": "cumulative"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:2095",
"format": "short",
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"$$hashKey": "object:2096",
"format": "short",
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": null,
"editable": true,
"error": false,
"fieldConfig": {
"defaults": {
"custom": {}
},
"overrides": []
},
"fill": 1,
"fillGradient": 0,
"grid": {},
"gridPos": {
"h": 7,
"w": 24,
"x": 0,
"y": 18
},
"hiddenSeries": false,
"id": 77,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 2,
"links": [],
"nullPointMode": "connected",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "7.4.2",
"pointradius": 2,
"points": true,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"refId": "A",
"target": "aliasByNode(highestMax(stats.$dc.gauges.hoodie.$Environment.$Dataset.$Tablefilter.clean.numFilesDeleted.count, 10), 1, 6)",
"textEditor": false
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "Number of OLD files deleted (Top 10)",
"tooltip": {
"shared": false,
"sort": 0,
"value_type": "cumulative"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:2185",
"format": "short",
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"$$hashKey": "object:2186",
"format": "short",
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": null,
"editable": true,
"error": false,
"fieldConfig": {
"defaults": {
"custom": {}
},
"overrides": []
},
"fill": 1,
"fillGradient": 0,
"grid": {},
"gridPos": {
"h": 7,
"w": 24,
"x": 0,
"y": 25
},
"hiddenSeries": false,
"id": 79,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 2,
"links": [],
"nullPointMode": "connected",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "7.4.2",
"pointradius": 2,
"points": true,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"refId": "A",
"target": "aliasByNode(highestMax(stats.$dc.gauges.hoodie.$Environment.$Dataset.$Tablefilter.commit.totalInsertRecordsWritten.count, 10), 1, 6)",
"textEditor": true
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "Number of NEW records Added (Top 10)",
"tooltip": {
"shared": false,
"sort": 0,
"value_type": "cumulative"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:2279",
"format": "short",
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"$$hashKey": "object:2280",
"format": "short",
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": null,
"editable": true,
"error": false,
"fieldConfig": {
"defaults": {
"custom": {}
},
"overrides": []
},
"fill": 1,
"fillGradient": 0,
"grid": {},
"gridPos": {
"h": 7,
"w": 24,
"x": 0,
"y": 32
},
"hiddenSeries": false,
"id": 78,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 2,
"links": [],
"nullPointMode": "connected",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "7.4.2",
"pointradius": 2,
"points": true,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"refId": "A",
"target": "aliasByNode(highestMax(stats.$dc.gauges.hoodie.$Environment.$Dataset.$Tablefilter.commit.totalUpdateRecordsWritten.count, 10), 1, 6)",
"textEditor": true
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "Number of OLD records updated (Top 10)",
"tooltip": {
"shared": false,
"sort": 0,
"value_type": "cumulative"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:2365",
"format": "short",
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"$$hashKey": "object:2366",
"format": "short",
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": null,
"editable": true,
"error": false,
"fieldConfig": {
"defaults": {
"custom": {}
},
"overrides": []
},
"fill": 1,
"fillGradient": 0,
"grid": {},
"gridPos": {
"h": 7,
"w": 24,
"x": 0,
"y": 39
},
"hiddenSeries": false,
"id": 75,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 2,
"links": [],
"nullPointMode": "connected",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "7.4.2",
"pointradius": 2,
"points": true,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"refId": "A",
"target": "aliasByNode(highestMax(stats.$dc.gauges.hoodie.$Environment.$Dataset.$Tablefilter.commit.totalRecordsWritten.count, 10), 1, 6)",
"textEditor": true
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "How many records (INSERT/UPDATE/COPY) written? (Top 10)",
"tooltip": {
"shared": false,
"sort": 0,
"value_type": "cumulative"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:2451",
"format": "short",
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"$$hashKey": "object:2452",
"format": "short",
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": null,
"editable": true,
"error": false,
"fieldConfig": {
"defaults": {
"custom": {}
},
"overrides": []
},
"fill": 1,
"fillGradient": 0,
"grid": {},
"gridPos": {
"h": 7,
"w": 24,
"x": 0,
"y": 46
},
"hiddenSeries": false,
"id": 74,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 2,
"links": [],
"nullPointMode": "connected",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "7.4.2",
"pointradius": 2,
"points": true,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"refId": "A",
"target": "aliasByNode(highestMax(stats.$dc.gauges.hoodie.$Environment.$Dataset.$Tablefilter.commit.totalPartitionsWritten.count, 10), 1, 6)",
"textEditor": false
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "How many partitions updated? (Top 10)",
"tooltip": {
"shared": false,
"sort": 0,
"value_type": "cumulative"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:2537",
"format": "short",
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"$$hashKey": "object:2538",
"format": "short",
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": null,
"editable": true,
"error": false,
"fieldConfig": {
"defaults": {
"custom": {}
},
"overrides": []
},
"fill": 1,
"fillGradient": 0,
"grid": {},
"gridPos": {
"h": 7,
"w": 24,
"x": 0,
"y": 53
},
"hiddenSeries": false,
"id": 76,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 2,
"links": [],
"nullPointMode": "connected",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "7.4.2",
"pointradius": 2,
"points": true,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"refId": "A",
"target": "aliasByNode(highestMax(stats.$dc.gauges.hoodie.$Environment.$Dataset.$Tablefilter.commit.totalBytesWritten.count, 10), 1, 6)",
"textEditor": true
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "How many bytes written? (Top 10)",
"tooltip": {
"shared": false,
"sort": 0,
"value_type": "cumulative"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:2623",
"format": "bytes",
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"$$hashKey": "object:2624",
"format": "short",
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
}
],
"title": "Dataset Detailed Metrics",
"type": "row"
},
{
"collapsed": true,
"datasource": null,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 4
},
"id": 104,
"panels": [
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "M3QL",
"description": "Count of various types of HDFS RPC calls for files outside the .hoodie folder",
"fieldConfig": {
"defaults": {
"custom": {}
},
"overrides": []
},
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 8,
"w": 8,
"x": 0,
"y": 5
},
"hiddenSeries": false,
"id": 106,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 1,
"nullPointMode": "connected",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "7.4.2",
"pointradius": 4,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"refId": "A",
"target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.HoodieWrapperFileSystem.create | removeEmpty | sumSeries | alias \"create\"\n"
},
{
"hide": false,
"refId": "B",
"target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.HoodieWrapperFileSystem.rename | removeEmpty | sumSeries | alias \"rename\"\n"
},
{
"hide": false,
"refId": "C",
"target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.HoodieWrapperFileSystem.delete | removeEmpty | sumSeries | alias \"delete\"\n"
},
{
"hide": false,
"refId": "D",
"target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.HoodieWrapperFileSystem.listStatus | removeEmpty | sumSeries | alias \"listStatus\"\n"
},
{
"hide": false,
"refId": "E",
"target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.HoodieWrapperFileSystem.mkdirs | removeEmpty | sumSeries | alias \"mkdirs\"\n"
},
{
"hide": false,
"refId": "F",
"target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.HoodieWrapperFileSystem.getFileStatus | removeEmpty | sumSeries | alias \"getFileStatus\"\n"
},
{
"hide": false,
"refId": "G",
"target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.HoodieWrapperFileSystem.globStatus | removeEmpty | sumSeries | alias \"globStatus\"\n"
},
{
"hide": false,
"refId": "H",
"target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.HoodieWrapperFileSystem.listFiles | removeEmpty | sumSeries | alias \"listFiles\"\n"
},
{
"hide": false,
"refId": "I",
"target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.HoodieWrapperFileSystem.open | removeEmpty | sumSeries | alias \"open\"\n"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "HDFS RPC Calls (outside .hoodie folder)",
"tooltip": {
"shared": true,
"sort": 0,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:5303",
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"$$hashKey": "object:5304",
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "M3QL",
"description": "Count of various types of HDFS RPC calls for files outside the .hoodie folder",
"fieldConfig": {
"defaults": {
"custom": {}
},
"overrides": []
},
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 8,
"w": 8,
"x": 8,
"y": 5
},
"hiddenSeries": false,
"id": 255,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 1,
"nullPointMode": "connected",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "7.4.2",
"pointradius": 4,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"refId": "A",
"target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.HoodieWrapperFileSystem.read | removeEmpty | sumSeries | alias \"read\"\n"
},
{
"hide": false,
"refId": "B",
"target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.HoodieWrapperFileSystem.write | removeEmpty | sumSeries | alias \"write\"\n"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "HDFS RPC Calls (outside .hoodie folder) - read / write",
"tooltip": {
"shared": true,
"sort": 0,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:5303",
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"$$hashKey": "object:5304",
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "M3QL",
"description": "Total bytes for read/writeHDFS RPC calls",
"fieldConfig": {
"defaults": {
"custom": {}
},
"overrides": []
},
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 8,
"w": 8,
"x": 16,
"y": 5
},
"hiddenSeries": false,
"id": 161,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": true,
"total": true,
"values": true
},
"lines": true,
"linewidth": 1,
"nullPointMode": "connected",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "7.4.2",
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"refId": "A",
"target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.HoodieWrapperFileSystem.read.totalBytes | removeEmpty | sumSeries | alias \"read\"\n"
},
{
"hide": false,
"refId": "B",
"target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.HoodieWrapperFileSystem.write.totalBytes | removeEmpty | sumSeries | alias \"write\"\n"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "HDFS Read / Write Bytes (outside .hoodie folder)",
"tooltip": {
"shared": true,
"sort": 0,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:5303",
"format": "bytes",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"$$hashKey": "object:5304",
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "M3QL",
"description": "Count of various types of HDFS RPC calls for files inside the .hoodie folder",
"fieldConfig": {
"defaults": {
"custom": {}
},
"overrides": []
},
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 8,
"w": 8,
"x": 0,
"y": 13
},
"hiddenSeries": false,
"id": 216,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 1,
"nullPointMode": "connected",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "7.4.2",
"pointradius": 4,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"refId": "A",
"target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.HoodieWrapperFileSystemMetaFolder.create | removeEmpty | sumSeries | alias \"create\"\n"
},
{
"hide": false,
"refId": "B",
"target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.HoodieWrapperFileSystemMetaFolder.rename | removeEmpty | sumSeries | alias \"rename\"\n"
},
{
"hide": false,
"refId": "C",
"target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.HoodieWrapperFileSystemMetaFolder.delete | removeEmpty | sumSeries | alias \"delete\"\n"
},
{
"hide": false,
"refId": "D",
"target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.HoodieWrapperFileSystemMetaFolder.listStatus | removeEmpty | sumSeries | alias \"listStatus\"\n"
},
{
"hide": false,
"refId": "E",
"target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.HoodieWrapperFileSystemMetaFolder.mkdirs | removeEmpty | sumSeries | alias \"mkdirs\"\n"
},
{
"hide": false,
"refId": "F",
"target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.HoodieWrapperFileSystemMetaFolder.getFileStatus | removeEmpty | sumSeries | alias \"getFileStatus\"\n"
},
{
"hide": false,
"refId": "G",
"target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.HoodieWrapperFileSystemMetaFolder.globStatus | removeEmpty | sumSeries | alias \"globStatus\"\n"
},
{
"hide": false,
"refId": "H",
"target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.HoodieWrapperFileSystemMetaFolder.listFiles | removeEmpty | sumSeries | alias \"listFiles\"\n"
},
{
"hide": false,
"refId": "I",
"target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.HoodieWrapperFileSystemMetaFolder.open | removeEmpty | sumSeries | alias \"open\"\n"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "HDFS RPC Calls (.hoodie folder)",
"tooltip": {
"shared": true,
"sort": 0,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:5303",
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"$$hashKey": "object:5304",
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "M3QL",
"description": "Count of various types of HDFS RPC calls for files inside the .hoodie folder",
"fieldConfig": {
"defaults": {
"custom": {}
},
"overrides": []
},
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 8,
"w": 8,
"x": 8,
"y": 13
},
"hiddenSeries": false,
"id": 254,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 1,
"nullPointMode": "connected",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "7.4.2",
"pointradius": 4,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"refId": "A",
"target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.HoodieWrapperFileSystemMetaFolder.read | removeEmpty | sumSeries | alias \"read\"\n"
},
{
"hide": false,
"refId": "B",
"target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.HoodieWrapperFileSystemMetaFolder.write | removeEmpty | sumSeries | alias \"write\"\n"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "HDFS RPC Calls (.hoodie folder) - read / write",
"tooltip": {
"shared": true,
"sort": 0,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:5303",
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"$$hashKey": "object:5304",
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "M3QL",
"description": "Total bytes for read/writeHDFS RPC calls within .hoodie folder",
"fieldConfig": {
"defaults": {
"custom": {}
},
"overrides": []
},
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 8,
"w": 8,
"x": 16,
"y": 13
},
"hiddenSeries": false,
"id": 160,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": true,
"total": true,
"values": true
},
"lines": true,
"linewidth": 1,
"nullPointMode": "connected",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "7.4.2",
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"refId": "A",
"target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.HoodieWrapperFileSystemMetaFolder.read.totalBytes | removeEmpty | sumSeries | alias \"read\"\n"
},
{
"hide": false,
"refId": "B",
"target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.HoodieWrapperFileSystemMetaFolder.write.totalBytes | removeEmpty | sumSeries | alias \"write\"\n"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "HDFS Read / Write Bytes (.hoodie folder)",
"tooltip": {
"shared": true,
"sort": 0,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:5303",
"format": "bytes",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"$$hashKey": "object:5304",
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "M3QL",
"description": "Average timings of various types of HDFS RPC calls",
"fieldConfig": {
"defaults": {
"custom": {}
},
"overrides": []
},
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 8,
"w": 8,
"x": 0,
"y": 21
},
"hiddenSeries": false,
"id": 110,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 1,
"nullPointMode": "connected",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "7.4.2",
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"refId": "A",
"target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.HoodieWrapperFileSystem*.delete.totalDuration | removeEmpty | sumSeries | divideSeries (fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.HoodieWrapperFileSystem*.delete | removeEmpty | sumSeries) | alias \"delete\""
},
{
"hide": false,
"refId": "B",
"target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.HoodieWrapperFileSystem*.create.totalDuration | removeEmpty | sumSeries | divideSeries (fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.HoodieWrapperFileSystem*.create | removeEmpty | sumSeries) | alias \"create\""
},
{
"hide": false,
"refId": "C",
"target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.HoodieWrapperFileSystem*.rename.totalDuration | removeEmpty | sumSeries | divideSeries (fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.HoodieWrapperFileSystem*.rename | removeEmpty | sumSeries) | alias \"rename\""
},
{
"hide": false,
"refId": "D",
"target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.HoodieWrapperFileSystem*.listStatus.totalDuration | removeEmpty | sumSeries | divideSeries (fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.HoodieWrapperFileSystem*.listStatus | removeEmpty | sumSeries) | alias \"listStatus\""
},
{
"hide": false,
"refId": "E",
"target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.HoodieWrapperFileSystem*.mkdirs.totalDuration | removeEmpty | sumSeries | divideSeries (fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.HoodieWrapperFileSystem*.mkdirs | removeEmpty | sumSeries) | alias \"mkdirs\""
},
{
"hide": false,
"refId": "F",
"target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.HoodieWrapperFileSystem*.getFileStatus.totalDuration | removeEmpty | sumSeries | divideSeries (fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.HoodieWrapperFileSystem*.getFileStatus | removeEmpty | sumSeries) | alias \"getFileStatus\""
},
{
"hide": false,
"refId": "G",
"target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.HoodieWrapperFileSystem*.globStatus.totalDuration | removeEmpty | sumSeries | divideSeries (fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.HoodieWrapperFileSystem*.globStatus | removeEmpty | sumSeries) | alias \"globStatus\""
},
{
"hide": false,
"refId": "H",
"target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.HoodieWrapperFileSystem*.listFiles.totalDuration | removeEmpty | sumSeries | divideSeries (fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.HoodieWrapperFileSystem*.listFiles | removeEmpty | sumSeries) | alias \"listFiles\""
},
{
"hide": false,
"refId": "I",
"target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.HoodieWrapperFileSystem*.read.totalDuration | removeEmpty | sumSeries | divideSeries (fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.HoodieWrapperFileSystem*.read | removeEmpty | sumSeries) | alias \"read\""
},
{
"hide": false,
"refId": "J",
"target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.HoodieWrapperFileSystem*.write.totalDuration | removeEmpty | sumSeries | divideSeries (fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.HoodieWrapperFileSystem*.write | removeEmpty | sumSeries) | alias \"write\""
},
{
"hide": false,
"refId": "K",
"target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.HoodieWrapperFileSystem*.open.totalDuration | removeEmpty | sumSeries | divideSeries (fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.HoodieWrapperFileSystem*.open | removeEmpty | sumSeries) | alias \"open\""
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "HDFS RPC Calls Timings",
"tooltip": {
"shared": true,
"sort": 0,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:5303",
"format": "ms",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"$$hashKey": "object:5304",
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
}
],
"title": "Hadoop & RPC Metrics",
"type": "row"
},
{
"collapsed": true,
"datasource": null,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 5
},
"id": 195,
"panels": [
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "M3QL",
"description": "Counts of of various types of TimelineService API calls",
"fieldConfig": {
"defaults": {
"custom": {}
},
"overrides": []
},
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 8,
"w": 8,
"x": 0,
"y": 6
},
"hiddenSeries": false,
"id": 177,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 1,
"nullPointMode": "connected",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "7.4.2",
"pointradius": 3,
"points": true,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"refId": "A",
"target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.TimelineService.ALL_FILEGROUPS_FOR_PARTITION | removeEmpty | sum | alias \"ALL_FILEGROUPS_FOR_PARTITION\"\n"
},
{
"hide": false,
"refId": "B",
"target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.TimelineService.ALL_REPLACED_FILEGROUPS_PARTITION | removeEmpty | sum | alias \"ALL_REPLACED_FILEGROUPS_PARTITION\"\n"
},
{
"hide": false,
"refId": "C",
"target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.TimelineService.LATEST_DATA_FILES_BEFORE_ON_INSTANT | removeEmpty | sum | alias \"LATEST_DATA_FILES_BEFORE_ON_INSTANT\"\n"
},
{
"hide": false,
"refId": "D",
"target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.TimelineService.LATEST_PARTITION_DATA_FILE | removeEmpty | sum | alias \"LATEST_PARTITION_DATA_FILE\"\n"
},
{
"hide": false,
"refId": "E",
"target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.TimelineService.PEDING_COMPACTION_OPS | removeEmpty | sum | alias \"PEDING_COMPACTION_OPS\"\n"
},
{
"hide": false,
"refId": "F",
"target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.TimelineService.LATEST_ALL_DATA_FILES | removeEmpty | sum | alias \"LATEST_ALL_DATA_FILES\"\n"
},
{
"hide": false,
"refId": "G",
"target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.TimelineService.LATEST_DATA_FILE_ON_INSTANT | removeEmpty | sum | alias \"LATEST_DATA_FILE_ON_INSTANT\"\n"
},
{
"hide": false,
"refId": "H",
"target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.TimelineService.ALL_DATA_FILES | removeEmpty | sum | alias \"ALL_DATA_FILES\"\n"
},
{
"hide": false,
"refId": "I",
"target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.TimelineService.LATEST_DATA_FILES_RANGE_INSTANT | removeEmpty | sum | alias \"LATEST_DATA_FILES_RANGE_INSTANT\"\n"
},
{
"hide": true,
"refId": "J",
"target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.TimelineService.* | removeEmpty \n"
},
{
"hide": false,
"refId": "K",
"target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.TimelineService.ALL_REPLACED_FILEGROUPS_BEFORE | removeEmpty | sum | alias \"ALL_REPLACED_FILEGROUPS_BEFORE\"\n"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "Total API Calls",
"tooltip": {
"shared": true,
"sort": 0,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:5303",
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"$$hashKey": "object:5304",
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "M3QL",
"description": "Timings of various types of TimelineService API calls",
"fieldConfig": {
"defaults": {
"custom": {}
},
"overrides": []
},
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 8,
"w": 8,
"x": 8,
"y": 6
},
"hiddenSeries": false,
"id": 162,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 1,
"nullPointMode": "connected",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "7.4.2",
"pointradius": 3,
"points": true,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"refId": "A",
"target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.TimelineService.TOTAL_API_TIME | removeEmpty | sumSeries | divideSeries (fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.TimelineService.TOTAL_API_CALLS | removeEmpty | sumSeries) | alias \"API Time\""
},
{
"hide": false,
"refId": "B",
"target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.TimelineService.TOTAL_CHECK_TIME | removeEmpty | sumSeries | divideSeries (fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.TimelineService.TOTAL_API_CALLS | removeEmpty | sumSeries) | alias \"Check Time\""
},
{
"hide": false,
"refId": "C",
"target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.TimelineService.TOTAL_REFRESH_TIME | removeEmpty | sumSeries | divideSeries (fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.TimelineService.TOTAL_API_CALLS | removeEmpty | sumSeries) | alias \"Refresh Time\""
},
{
"hide": false,
"refId": "D",
"target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.TimelineService.TOTAL_HANDLE_TIME | removeEmpty | sumSeries | divideSeries (fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.TimelineService.TOTAL_API_CALLS | removeEmpty | sumSeries) | alias \"Handle Time\""
},
{
"hide": false,
"refId": "E",
"target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.TimelineService.WRITE_VALUE_TIME | removeEmpty | sumSeries | divideSeries (fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.TimelineService.WRITE_VALUE_CNT | removeEmpty | sumSeries) | alias \"Json Encoding Time\""
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "Timing for operations (Avg)",
"tooltip": {
"shared": true,
"sort": 0,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:5303",
"format": "ms",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"$$hashKey": "object:5304",
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "M3QL",
"description": "Timings of various types of TimelineService API calls",
"fieldConfig": {
"defaults": {
"custom": {}
},
"overrides": []
},
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 8,
"w": 8,
"x": 16,
"y": 6
},
"hiddenSeries": false,
"id": 219,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 1,
"nullPointMode": "connected",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "7.4.2",
"pointradius": 3,
"points": true,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"refId": "A",
"target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.TimelineService.TOTAL_API_TIME | removeEmpty | sumSeries | divideSeries (fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.TimelineService.TOTAL_API_CALLS | removeEmpty | sumSeries) | nPercentile 95 | alias \"TimeLine Service P95\""
},
{
"hide": false,
"refId": "B",
"target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.HoodieWrapperFileSystem*.listStatus.totalDuration | removeEmpty | sumSeries | divideSeries (fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.HoodieWrapperFileSystem*.listStatus | removeEmpty | sumSeries) | nPercentile 95 | alias \"HDFS\""
},
{
"hide": false,
"refId": "C",
"target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.TimelineService.TOTAL_API_TIME | removeEmpty | sumSeries | moving 1D sum | divideSeries (fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.TimelineService.TOTAL_API_CALLS | removeEmpty | sumSeries | moving 1D sum) | alias \"TimeLine Service AVG\""
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "TimelineService API vs HDFS RPC Time",
"tooltip": {
"shared": true,
"sort": 0,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:5303",
"format": "ms",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"$$hashKey": "object:5304",
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
}
],
"title": "Timelines Metrics 0.7v",
"type": "row"
},
{
"collapsed": true,
"datasource": null,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 6
},
"id": 102,
"panels": [
{
"cacheTimeout": null,
"datasource": "M3QL",
"description": "Total number of RPC calls saved / day due to the use of Metadata Table",
"fieldConfig": {
"defaults": {
"custom": {},
"mappings": [
{
"id": 0,
"op": "=",
"text": "N/A",
"type": 1,
"value": "null"
}
],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
},
"unit": "short"
},
"overrides": []
},
"gridPos": {
"h": 6,
"w": 4,
"x": 0,
"y": 7
},
"id": 221,
"interval": null,
"links": [],
"maxDataPoints": 100,
"options": {
"colorMode": "value",
"graphMode": "none",
"justifyMode": "center",
"orientation": "horizontal",
"reduceOptions": {
"calcs": [
"mean"
],
"fields": "",
"values": false
},
"text": {},
"textMode": "value"
},
"pluginVersion": "7.4.2",
"targets": [
{
"refId": "A",
"target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.HoodieWrapperFileSystemMetaFolder.listStatus | sum | moving 1D sum\n"
},
{
"hide": true,
"refId": "B",
"target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.${Tablefilter}.HoodieMetadata.lookup_files.count | removeEmpty | sum | alias \"lookup_files\""
}
],
"title": "ListStatus Calls Required / Day (.hoodie folder)",
"type": "stat"
},
{
"cacheTimeout": null,
"datasource": "M3QL",
"description": "Total number of RPC calls saved / day due to the use of Metadata Table",
"fieldConfig": {
"defaults": {
"custom": {},
"mappings": [
{
"id": 0,
"op": "=",
"text": "0",
"type": 1,
"value": "null"
}
],
"noValue": "0",
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
},
"unit": "short"
},
"overrides": []
},
"gridPos": {
"h": 6,
"w": 4,
"x": 4,
"y": 7
},
"id": 218,
"interval": null,
"links": [],
"maxDataPoints": 100,
"options": {
"colorMode": "value",
"graphMode": "none",
"justifyMode": "center",
"orientation": "horizontal",
"reduceOptions": {
"calcs": [
"mean"
],
"fields": "",
"values": false
},
"text": {},
"textMode": "value"
},
"pluginVersion": "7.4.2",
"targets": [
{
"hide": false,
"refId": "B",
"target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.{HoodieMetadata,HoodieWrapperFileSystem}.{lookup_files.count,lookup_partitions.count,listStatus} | removeEmpty | sum | moving 1D sum\n"
}
],
"title": "ListStatus Calls Required / Day (outside .hoodie folder)",
"type": "stat"
},
{
"cacheTimeout": null,
"datasource": "M3QL",
"description": "Total number of RPC calls saved / day due to the use of Metadata Table",
"fieldConfig": {
"defaults": {
"custom": {},
"mappings": [
{
"id": 0,
"op": "=",
"text": "N/A",
"type": 1,
"value": "null"
}
],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
},
"unit": "short"
},
"overrides": []
},
"gridPos": {
"h": 6,
"w": 4,
"x": 8,
"y": 7
},
"id": 224,
"interval": null,
"links": [],
"maxDataPoints": 100,
"options": {
"colorMode": "value",
"graphMode": "none",
"justifyMode": "center",
"orientation": "horizontal",
"reduceOptions": {
"calcs": [
"mean"
],
"fields": "",
"values": false
},
"text": {},
"textMode": "value"
},
"pluginVersion": "7.4.2",
"targets": [
{
"refId": "A",
"target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.{HoodieWrapperFileSystemMetaFolder,HoodieWrapperFileSystem,HoodieMetadata}.{lookup_files.count,lookup_partitions.count,listStatus} | sum | moving 1D sum\n"
},
{
"hide": true,
"refId": "B",
"target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.${Tablefilter}.HoodieMetadata.lookup_files.count | removeEmpty | sum | alias \"lookup_files\""
}
],
"title": "Total HUDI listStatus Calls / Day",
"type": "stat"
},
{
"cacheTimeout": null,
"datasource": "M3QL",
"description": "Total number of RPC calls saved / day due to the use of Metadata Table",
"fieldConfig": {
"defaults": {
"custom": {},
"mappings": [
{
"id": 0,
"op": "=",
"text": "0",
"type": 1,
"value": "null"
}
],
"noValue": "0",
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
},
"unit": "short"
},
"overrides": []
},
"gridPos": {
"h": 6,
"w": 4,
"x": 12,
"y": 7
},
"id": 222,
"interval": null,
"links": [],
"maxDataPoints": 100,
"options": {
"colorMode": "value",
"graphMode": "none",
"justifyMode": "center",
"orientation": "horizontal",
"reduceOptions": {
"calcs": [
"mean"
],
"fields": "",
"values": false
},
"text": {},
"textMode": "value"
},
"pluginVersion": "7.4.2",
"targets": [
{
"refId": "A",
"target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.HoodieMetadata.{lookup_files,lookup_partitions}.count | sum | moving 1D sum\n"
},
{
"hide": true,
"refId": "B",
"target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.${Tablefilter}.HoodieMetadata.lookup_files.count | removeEmpty | sum | alias \"lookup_files\""
}
],
"title": "Total listStatus Calls Removed / Day",
"type": "stat"
},
{
"cacheTimeout": null,
"datasource": "M3QL",
"description": "Percent of listStatus Calls Removed / Day due to the use of Metadata Table",
"fieldConfig": {
"defaults": {
"color": {
"mode": "continuous-RdYlGr"
},
"custom": {},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": []
},
"unit": "percent"
},
"overrides": []
},
"gridPos": {
"h": 6,
"w": 4,
"x": 16,
"y": 7
},
"id": 223,
"interval": null,
"links": [],
"maxDataPoints": 100,
"options": {
"orientation": "horizontal",
"reduceOptions": {
"calcs": [
"mean"
],
"fields": "",
"values": false
},
"showThresholdLabels": true,
"showThresholdMarkers": true,
"text": {}
},
"pluginVersion": "7.4.2",
"targets": [
{
"hide": false,
"refId": "B",
"target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.HoodieMetadata.{lookup_files.count,lookup_partitions.count} | sum | moving 1D sum | divideSeries (fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.{HoodieWrapperFileSystemMetaFolder,HoodieWrapperFileSystem,HoodieMetadata}.{lookup_files.count,lookup_partitions.count,listStatus} | removeEmpty | sum | moving 1D sum ) | scale 100\n"
}
],
"title": "% of listStatus Calls Removed / Day (of Total)",
"type": "gauge"
},
{
"cacheTimeout": null,
"datasource": "M3QL",
"description": "Percent of listStatus Calls Removed / Day due to the use of Metadata Table as implemented in current production release of HUDI.",
"fieldConfig": {
"defaults": {
"color": {
"mode": "continuous-RdYlGr"
},
"custom": {},
"mappings": [
{
"id": 0,
"op": "=",
"text": "0",
"type": 1,
"value": "null"
}
],
"max": 100,
"min": 0,
"noValue": "?",
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
},
"unit": "percent"
},
"overrides": []
},
"gridPos": {
"h": 6,
"w": 4,
"x": 20,
"y": 7
},
"id": 226,
"interval": null,
"links": [],
"maxDataPoints": 100,
"options": {
"orientation": "horizontal",
"reduceOptions": {
"calcs": [
"mean"
],
"fields": "",
"values": false
},
"showThresholdLabels": true,
"showThresholdMarkers": true,
"text": {}
},
"pluginVersion": "7.4.2",
"targets": [
{
"refId": "A",
"target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.HoodieMetadata.{lookup_files.count,lookup_partitions.count} | sum | divideSeries (fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.{HoodieWrapperFileSystem,HoodieMetadata}.{lookup_files.count,lookup_partitions.count,listStatus} | sum) | scale 100\n"
}
],
"title": "% of listStatus Calls Removed / Day (As Implemented)",
"type": "gauge"
},
{
"datasource": "M3QL",
"description": "Maximum Partitions written in each DeltaCommit (should be 1)",
"fieldConfig": {
"defaults": {
"custom": {},
"mappings": [],
"noValue": "0",
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 6,
"w": 4,
"x": 0,
"y": 13
},
"id": 136,
"links": [],
"options": {
"colorMode": "value",
"graphMode": "none",
"justifyMode": "auto",
"orientation": "auto",
"reduceOptions": {
"calcs": [
"max"
],
"fields": "",
"values": false
},
"text": {},
"textMode": "auto"
},
"pluginVersion": "7.4.2",
"targets": [
{
"refId": "A",
"target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.${Tablefilter}_metadata.deltacommit.totalPartitionsWritten | max\n"
}
],
"timeFrom": null,
"timeShift": null,
"title": "DeltaCommit - Max Partitions Written",
"type": "stat"
},
{
"datasource": "M3QL",
"description": "Maximum files updated in each DeltaCommit (should be 1)",
"fieldConfig": {
"defaults": {
"custom": {},
"mappings": [],
"noValue": "0",
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 6,
"w": 4,
"x": 4,
"y": 13
},
"id": 138,
"links": [],
"options": {
"colorMode": "value",
"graphMode": "none",
"justifyMode": "auto",
"orientation": "auto",
"reduceOptions": {
"calcs": [
"max"
],
"fields": "",
"values": false
},
"text": {},
"textMode": "auto"
},
"pluginVersion": "7.4.2",
"targets": [
{
"refId": "A",
"target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.${Tablefilter}_metadata.deltacommit.totalFilesUpdate | max\n"
}
],
"timeFrom": null,
"timeShift": null,
"title": "DeltaCommit - Max Files Updated",
"type": "stat"
},
{
"datasource": "M3QL",
"description": "Maximum records written in a Deltacommit",
"fieldConfig": {
"defaults": {
"custom": {},
"mappings": [],
"noValue": "0",
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 6,
"w": 4,
"x": 8,
"y": 13
},
"id": 140,
"links": [],
"options": {
"colorMode": "value",
"graphMode": "none",
"justifyMode": "auto",
"orientation": "auto",
"reduceOptions": {
"calcs": [
"max"
],
"fields": "",
"values": false
},
"text": {},
"textMode": "auto"
},
"pluginVersion": "7.4.2",
"targets": [
{
"refId": "A",
"target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.${Tablefilter}_metadata.deltacommit.totalRecordsWritten | max\n"
}
],
"timeFrom": null,
"timeShift": null,
"title": "DeltaCommit - Max records written",
"type": "stat"
},
{
"datasource": "M3QL",
"description": "Maximum Partitions written in each Compaction (should be 1)",
"fieldConfig": {
"defaults": {
"custom": {},
"mappings": [],
"noValue": "0",
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 6,
"w": 4,
"x": 12,
"y": 13
},
"id": 142,
"links": [],
"options": {
"colorMode": "value",
"graphMode": "none",
"justifyMode": "auto",
"orientation": "auto",
"reduceOptions": {
"calcs": [
"max"
],
"fields": "",
"values": false
},
"text": {},
"textMode": "auto"
},
"pluginVersion": "7.4.2",
"targets": [
{
"refId": "A",
"target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.${Tablefilter}_metadata.compaction.totalPartitionsWritten | max\n"
}
],
"timeFrom": null,
"timeShift": null,
"title": "Compaction - Max Partitions Written",
"type": "stat"
},
{
"datasource": "M3QL",
"description": "Maximum log files compacted in each Compaction (should be 1)",
"fieldConfig": {
"defaults": {
"custom": {},
"mappings": [],
"noValue": "0",
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 6,
"w": 4,
"x": 16,
"y": 13
},
"id": 144,
"links": [],
"options": {
"colorMode": "value",
"graphMode": "none",
"justifyMode": "auto",
"orientation": "auto",
"reduceOptions": {
"calcs": [
"max"
],
"fields": "",
"values": false
},
"text": {},
"textMode": "auto"
},
"pluginVersion": "7.4.2",
"targets": [
{
"refId": "A",
"target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.${Tablefilter}_metadata.compaction.totalLogFilesCompacted | max\n"
}
],
"timeFrom": null,
"timeShift": null,
"title": "Compaction - Max LogFiles Compacted",
"type": "stat"
},
{
"datasource": "M3QL",
"description": "Maximum records and partitions written in a Compaction",
"fieldConfig": {
"defaults": {
"custom": {},
"mappings": [],
"noValue": "0",
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 6,
"w": 4,
"x": 20,
"y": 13
},
"id": 146,
"links": [],
"options": {
"colorMode": "value",
"graphMode": "area",
"justifyMode": "center",
"orientation": "auto",
"reduceOptions": {
"calcs": [
"max"
],
"fields": "",
"values": false
},
"text": {},
"textMode": "value"
},
"pluginVersion": "7.4.2",
"targets": [
{
"refId": "A",
"target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.${Tablefilter}_metadata.compaction.totalRecordsWritten | max\n"
}
],
"timeFrom": null,
"timeShift": null,
"title": "Compaction - Max Records Written",
"type": "stat"
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "M3QL",
"description": "Total time taken for various operations on the metadata table",
"fieldConfig": {
"defaults": {
"custom": {}
},
"overrides": []
},
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 9,
"w": 11,
"x": 0,
"y": 19
},
"hiddenSeries": false,
"id": 112,
"legend": {
"avg": false,
"current": false,
"hideEmpty": true,
"max": false,
"min": false,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 1,
"links": [],
"nullPointMode": "connected",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "7.4.2",
"pointradius": 1,
"points": true,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"hide": false,
"refId": "A",
"target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.${Tablefilter}.HoodieMetadata.sync.totalDuration | removeEmpty | sum | divideSeries (fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.${Tablefilter}.HoodieMetadata.sync.count | removeEmpty | sum ) | alias \"sync\"\n"
},
{
"hide": false,
"refId": "B",
"target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.${Tablefilter}.HoodieMetadata.scan.totalDuration | removeEmpty | sum | divideSeries (fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.${Tablefilter}.HoodieMetadata.scan.count | removeEmpty | sum ) | alias \"scan\"\n"
},
{
"hide": false,
"refId": "C",
"target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.${Tablefilter}.HoodieMetadata.initialize.totalDuration | removeEmpty | sum | divideSeries (fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.${Tablefilter}.HoodieMetadata.initialize.count | removeEmpty | sum ) | alias \"initialize\"\n"
},
{
"hide": false,
"refId": "D",
"target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.${Tablefilter}.HoodieMetadata.lookup_partitions.totalDuration | removeEmpty | sum | divideSeries (fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.${Tablefilter}.HoodieMetadata.lookup_partitions.count | removeEmpty | sum ) | alias \"lookup_partitions\"\n"
},
{
"hide": false,
"refId": "E",
"target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.${Tablefilter}.HoodieMetadata.lookup_files.totalDuration | removeEmpty | sum | divideSeries (fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.${Tablefilter}.HoodieMetadata.lookup_files.count | removeEmpty | sum ) | alias \"lookup_files\"\n"
},
{
"hide": false,
"refId": "F",
"target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.${Tablefilter}.HoodieMetadata.basefile_read.totalDuration | removeEmpty | sum | divideSeries (fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.${Tablefilter}.HoodieMetadata.basefile_read.count | removeEmpty | sum ) | alias \"basefile_read\"\n"
},
{
"hide": false,
"refId": "G",
"target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.${Tablefilter}.HoodieMetadata.validate_partitions.totalDuration | removeEmpty | sum | divideSeries (fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.${Tablefilter}.HoodieMetadata.validate_partitions.count | removeEmpty | sum ) | alias \"validate_partitions\"\n"
},
{
"hide": false,
"refId": "H",
"target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.${Tablefilter}.HoodieMetadata.validate_files.totalDuration | removeEmpty | sum | divideSeries (fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.${Tablefilter}.HoodieMetadata.validate_files.count | removeEmpty | sum ) | alias \"validate_files\"\n"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "Timings for various operations",
"tooltip": {
"shared": true,
"sort": 0,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:5946",
"format": "ms",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"$$hashKey": "object:5947",
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": false
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "M3QL",
"description": "Number of lookups for list of partitions or files within a partition from the metadata table",
"fieldConfig": {
"defaults": {
"custom": {}
},
"overrides": []
},
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 9,
"w": 13,
"x": 11,
"y": 19
},
"hiddenSeries": false,
"id": 114,
"legend": {
"avg": false,
"current": false,
"hideEmpty": true,
"max": false,
"min": false,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 1,
"links": [],
"nullPointMode": "connected",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "7.4.2",
"pointradius": 1,
"points": true,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"refId": "A",
"target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.${Tablefilter}.HoodieMetadata.lookup_partitions.count | removeEmpty | sum | alias \"lookup_partitions\""
},
{
"hide": false,
"refId": "B",
"target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.${Tablefilter}.HoodieMetadata.lookup_files.count | removeEmpty | sum | alias \"lookup_files\""
},
{
"hide": false,
"refId": "C",
"target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.${Tablefilter}.HoodieMetadata.sync.count | removeEmpty | sum | alias \"sync\""
},
{
"hide": false,
"refId": "D",
"target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.${Tablefilter}.HoodieMetadata.scan.count | removeEmpty | sum | alias \"scan\""
},
{
"hide": false,
"refId": "E",
"target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.${Tablefilter}.HoodieMetadata.initialize.count | removeEmpty | sum | alias \"initialize\""
},
{
"hide": false,
"refId": "F",
"target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.${Tablefilter}.HoodieMetadata.basefile_read.count | removeEmpty | sum | alias \"basefile_read\""
},
{
"hide": false,
"refId": "G",
"target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.${Tablefilter}.HoodieMetadata.validate_partitions.count | removeEmpty | sum | alias \"validate_partitions\""
},
{
"hide": false,
"refId": "H",
"target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.${Tablefilter}.HoodieMetadata.validate_files.count | removeEmpty | sum | alias \"validate_files\""
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "Counts of various operations ",
"tooltip": {
"shared": true,
"sort": 0,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:5421",
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"$$hashKey": "object:5422",
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": false
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "M3QL",
"description": "Total size of all metadata base files (HFIle) created",
"fieldConfig": {
"defaults": {
"custom": {}
},
"overrides": []
},
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 7,
"w": 6,
"x": 0,
"y": 28
},
"hiddenSeries": false,
"id": 124,
"legend": {
"avg": false,
"current": false,
"hideEmpty": true,
"max": false,
"min": false,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 1,
"links": [],
"nullPointMode": "connected",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "7.4.2",
"pointradius": 5,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"refId": "A",
"target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.${Tablefilter}.HoodieMetadata.basefile.size | removeEmpty | sum name | aliasByTags name\n"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "Metadata Basefile Size",
"tooltip": {
"shared": true,
"sort": 0,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:238",
"format": "bytes",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"$$hashKey": "object:239",
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": false
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "M3QL",
"description": "Number of metadata base files (HFIle) created",
"fieldConfig": {
"defaults": {
"custom": {}
},
"overrides": []
},
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 7,
"w": 7,
"x": 6,
"y": 28
},
"hiddenSeries": false,
"id": 126,
"legend": {
"avg": false,
"current": false,
"hideEmpty": true,
"max": false,
"min": false,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 3,
"links": [],
"nullPointMode": "connected",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "7.4.2",
"pointradius": 0.5,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"refId": "A",
"target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.${Tablefilter}.HoodieMetadata.basefile.count | removeEmpty | sum name | aliasByTags name\n"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "Metadata Basefile Count",
"tooltip": {
"shared": true,
"sort": 0,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:172",
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"$$hashKey": "object:173",
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": false
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "M3QL",
"description": "Total size of all metadata log files created",
"fieldConfig": {
"defaults": {
"custom": {}
},
"overrides": []
},
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 7,
"w": 6,
"x": 13,
"y": 28
},
"hiddenSeries": false,
"id": 128,
"legend": {
"avg": false,
"current": false,
"hideEmpty": true,
"max": false,
"min": false,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 1,
"links": [],
"nullPointMode": "connected",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "7.4.2",
"pointradius": 0.5,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"refId": "A",
"target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.${Tablefilter}.HoodieMetadata.logfile.size | removeEmpty | sum name | aliasByTags name\n"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "Metadata Logfile Size",
"tooltip": {
"shared": true,
"sort": 0,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:106",
"format": "bytes",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"$$hashKey": "object:107",
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": false
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "M3QL",
"description": "Number of metadata log files created",
"fieldConfig": {
"defaults": {
"custom": {}
},
"overrides": []
},
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 7,
"w": 5,
"x": 19,
"y": 28
},
"hiddenSeries": false,
"id": 130,
"legend": {
"avg": false,
"current": false,
"hideEmpty": true,
"max": false,
"min": false,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 1,
"links": [],
"nullPointMode": "connected",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "7.4.2",
"pointradius": 5,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"refId": "A",
"target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.${Tablefilter}.HoodieMetadata.logfile.count | removeEmpty | sum name | aliasByTags name\n"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "Metadata Logfile Count",
"tooltip": {
"shared": true,
"sort": 0,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:343",
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"$$hashKey": "object:344",
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": false
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"aliasColors": {
"ONCALL_LINE": "#BF1B00"
},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "M3QL",
"description": "Maximum Time taken for various actions on the Metadata Table",
"editable": true,
"error": false,
"fieldConfig": {
"defaults": {
"custom": {}
},
"overrides": []
},
"fill": 0,
"fillGradient": 0,
"grid": {},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 35
},
"hiddenSeries": false,
"id": 94,
"legend": {
"avg": true,
"current": false,
"max": false,
"min": false,
"show": true,
"total": false,
"values": true
},
"lines": true,
"linewidth": 1,
"links": [],
"nullPointMode": "connected",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "7.4.2",
"pointradius": 1,
"points": true,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"refId": "A",
"target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.${Tablefilter}_metadata.deltacommit.duration | removeEmpty | avg | alias \"deltacommit\"\n"
},
{
"hide": false,
"refId": "B",
"target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.${Tablefilter}_metadata.compaction.duration | removeEmpty | avg | alias \"compaction\"\n"
},
{
"hide": false,
"refId": "C",
"target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.${Tablefilter}_metadata.finalize.duration | removeEmpty | avg | alias \"finalize\"\n"
},
{
"hide": false,
"refId": "D",
"target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.${Tablefilter}_metadata.clean.duration | removeEmpty | avg | alias \"clean\"\n"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "Metadata Table Actions Duration (Average)",
"tooltip": {
"shared": false,
"sort": 0,
"value_type": "cumulative"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:2753",
"format": "ms",
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"$$hashKey": "object:2754",
"format": "short",
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"aliasColors": {
"ONCALL_LINE": "#BF1B00"
},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "M3QL",
"description": "Maximum Time taken for various actions on the Metadata Table",
"editable": true,
"error": false,
"fieldConfig": {
"defaults": {
"custom": {}
},
"overrides": []
},
"fill": 0,
"fillGradient": 0,
"grid": {},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 35
},
"hiddenSeries": false,
"id": 217,
"legend": {
"avg": true,
"current": false,
"max": false,
"min": false,
"show": true,
"total": false,
"values": true
},
"lines": true,
"linewidth": 1,
"links": [],
"nullPointMode": "connected",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "7.4.2",
"pointradius": 1,
"points": true,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"refId": "A",
"target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.${Tablefilter}_metadata.deltacommit.duration | removeEmpty | max | alias \"deltacommit\"\n"
},
{
"hide": false,
"refId": "B",
"target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.${Tablefilter}_metadata.compaction.duration | removeEmpty | max | alias \"compaction\"\n"
},
{
"hide": false,
"refId": "C",
"target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.${Tablefilter}_metadata.finalize.duration | removeEmpty | max | alias \"finalize\"\n"
},
{
"hide": false,
"refId": "D",
"target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.${Tablefilter}_metadata.clean.duration | removeEmpty | max | alias \"clean\"\n"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "Metadata Table Actions Duration (Max)",
"tooltip": {
"shared": false,
"sort": 0,
"value_type": "cumulative"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:2753",
"format": "ms",
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"$$hashKey": "object:2754",
"format": "short",
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "M3QL",
"description": "Number of error within the metadata table. ",
"fieldConfig": {
"defaults": {
"custom": {}
},
"overrides": []
},
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 7,
"w": 8,
"x": 0,
"y": 43
},
"hiddenSeries": false,
"id": 118,
"legend": {
"avg": false,
"current": false,
"hideEmpty": true,
"max": false,
"min": false,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 1,
"links": [],
"nullPointMode": "connected",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "7.4.2",
"pointradius": 1,
"points": true,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"refId": "A",
"target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.${Tablefilter}.HoodieMetadata.validate_errors.count | removeEmpty | sum name | alias \"validate_errors\""
},
{
"hide": false,
"refId": "B",
"target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.${Tablefilter}.HoodieMetadata.bootstrap_error.count | removeEmpty | sum name | alias \"bootstrap_errors\""
},
{
"hide": false,
"refId": "C",
"target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.${Tablefilter}.HoodieMetadata.rebootstrap.count | removeEmpty | sum name | alias \"rebootstrap\""
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "Metadata Table Errors",
"tooltip": {
"shared": true,
"sort": 0,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:5760",
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"$$hashKey": "object:5761",
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": false
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"aliasColors": {
"ONCALL_LINE": "#BF1B00"
},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "M3QL",
"description": "Records written to Metadata Table in each operation",
"editable": true,
"error": false,
"fieldConfig": {
"defaults": {
"custom": {}
},
"overrides": []
},
"fill": 0,
"fillGradient": 0,
"grid": {},
"gridPos": {
"h": 7,
"w": 8,
"x": 8,
"y": 43
},
"hiddenSeries": false,
"id": 100,
"legend": {
"avg": false,
"current": false,
"max": true,
"min": false,
"show": true,
"total": false,
"values": true
},
"lines": true,
"linewidth": 1,
"links": [],
"nullPointMode": "connected",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "7.4.2",
"pointradius": 1,
"points": true,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"refId": "A",
"target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.${Tablefilter}_metadata.deltacommit.totalInsertRecordsWritten | removeEmpty | sum | alias \"deltacommit.inserts\"\n"
},
{
"hide": false,
"refId": "B",
"target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.${Tablefilter}_metadata.deltacommit.totalUpdateRecordsWritten | removeEmpty | sum | alias \"deltacommit.updates\"\n"
},
{
"hide": false,
"refId": "C",
"target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.${Tablefilter}_metadata.deltacommit.totalRecordsWritten | removeEmpty | sum | alias \"deltacommit.total_records\"\n"
},
{
"hide": false,
"refId": "D",
"target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.${Tablefilter}_metadata.compaction.totalRecordsWritten | removeEmpty | sum | alias \"compaction.total_records\"\n"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "Metadata Table Record Writes",
"tooltip": {
"shared": false,
"sort": 0,
"value_type": "cumulative"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:2753",
"format": "short",
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"$$hashKey": "object:2754",
"format": "short",
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
}
],
"title": "Metadata Metrics (RFC-15)",
"type": "row"
},
{
"collapsed": true,
"datasource": null,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 7
},
"id": 198,
"panels": [
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "M3QL",
"description": "Number of HBase Inserts",
"fieldConfig": {
"defaults": {
"custom": {}
},
"overrides": []
},
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 9,
"w": 12,
"x": 0,
"y": 8
},
"hiddenSeries": false,
"id": 200,
"legend": {
"alignAsTable": true,
"avg": false,
"current": true,
"hideEmpty": true,
"hideZero": true,
"max": true,
"min": true,
"show": true,
"total": false,
"values": true
},
"lines": true,
"linewidth": 2,
"links": [],
"nullPointMode": "connected",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "7.4.2",
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": true,
"steppedLine": false,
"targets": [
{
"hide": true,
"refId": "A",
"target": " fetch service:hoodie datafeed:* name:hbase_num_puts env:production dc:dca* owner:* frequency:* source:* | sumSeries datafeed | sortSeries current desc | aliasByTags datafeed dc | removeEmpty"
},
{
"hide": false,
"refId": "B",
"target": " fetch service:hoodie datafeed:* name:hbase_num_puts env:production dc:dca* owner:* frequency:* source:* | sumSeries datafeed | sortSeries current desc | aliasByTags datafeed dc | removeEmpty | averageSeries | alias \"Average\""
},
{
"hide": false,
"refId": "C",
"target": " fetch service:hoodie datafeed:* name:hbase_num_puts env:production dc:dca* owner:* frequency:* source:* | sumSeries datafeed | sortSeries current desc | aliasByTags datafeed dc | removeEmpty | minSeries | alias \"Min\""
},
{
"hide": false,
"refId": "D",
"target": " fetch service:hoodie datafeed:* name:hbase_num_puts env:production dc:dca* owner:* frequency:* source:* | sumSeries datafeed | sortSeries current desc | aliasByTags datafeed dc | removeEmpty | maxSeries | alias \"Max\""
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "Number of HBase Inserts",
"tooltip": {
"shared": true,
"sort": 0,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:844",
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"$$hashKey": "object:845",
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "M3QL",
"description": "Total number of HBase Inserts",
"fieldConfig": {
"defaults": {
"custom": {}
},
"overrides": []
},
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 9,
"w": 12,
"x": 12,
"y": 8
},
"hiddenSeries": false,
"id": 201,
"legend": {
"alignAsTable": true,
"avg": true,
"current": true,
"hideEmpty": true,
"hideZero": true,
"max": true,
"min": true,
"show": true,
"total": false,
"values": true
},
"lines": true,
"linewidth": 2,
"links": [],
"nullPointMode": "connected",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "7.4.2",
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": true,
"steppedLine": false,
"targets": [
{
"hide": false,
"refId": "E",
"target": " fetch service:hoodie datafeed:* name:hbase_num_puts env:production dc:dca* owner:* frequency:* source:* | sumSeries | summarize 15m | alias \"Total\" | removeEmpty"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "Total HBase Inserts (Summed 15m interval)",
"tooltip": {
"shared": true,
"sort": 0,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:844",
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"$$hashKey": "object:845",
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "M3QL",
"fieldConfig": {
"defaults": {
"custom": {}
},
"overrides": []
},
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 9,
"w": 12,
"x": 0,
"y": 17
},
"hiddenSeries": false,
"id": 203,
"legend": {
"alignAsTable": true,
"avg": true,
"current": true,
"hideEmpty": true,
"hideZero": true,
"max": false,
"min": false,
"show": true,
"total": false,
"values": true
},
"lines": true,
"linewidth": 2,
"links": [],
"nullPointMode": "null",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "7.4.2",
"pointradius": 5,
"points": true,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"hide": true,
"refId": "A",
"target": "fetch service:hoodie datafeed:* name:hbase_qps_resources_actual_puts_time_secs env:production dc:* owner:* frequency:* source:* | sumSeries datafeed | sortSeries current desc | aliasByTags datafeed dc | removeEmpty"
},
{
"hide": false,
"refId": "C",
"target": "fetch service:hoodie datafeed:* name:hbase_qps_resources_actual_puts_time_secs env:production dc:* owner:* frequency:* source:* | sumSeries datafeed | sortSeries current desc | aliasByTags datafeed dc | removeEmpty | averageSeries | alias \"Average\""
},
{
"hide": false,
"refId": "B",
"target": "fetch service:hoodie datafeed:* name:hbase_qps_resources_actual_puts_time_secs env:production dc:* owner:* frequency:* source:* | sumSeries datafeed | sortSeries current desc | aliasByTags datafeed dc | removeEmpty | minSeries | alias \"Min\""
},
{
"hide": false,
"refId": "D",
"target": "fetch service:hoodie datafeed:* name:hbase_qps_resources_actual_puts_time_secs env:production dc:* owner:* frequency:* source:* | sumSeries datafeed | sortSeries current desc | aliasByTags datafeed dc | removeEmpty | maxSeries | alias \"Max\""
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "HBase puts time taken",
"tooltip": {
"shared": true,
"sort": 0,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:1153",
"format": "s",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"$$hashKey": "object:1154",
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "M3QL",
"fieldConfig": {
"defaults": {
"custom": {}
},
"overrides": []
},
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 9,
"w": 12,
"x": 12,
"y": 17
},
"hiddenSeries": false,
"id": 204,
"legend": {
"alignAsTable": true,
"avg": true,
"current": true,
"hideEmpty": true,
"hideZero": true,
"max": false,
"min": false,
"show": true,
"total": false,
"values": true
},
"lines": true,
"linewidth": 2,
"links": [],
"nullPointMode": "null",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "7.4.2",
"pointradius": 5,
"points": true,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"hide": false,
"refId": "A",
"target": "fetch service:hoodie datafeed:* name:hbase_qps_resources_actual_puts_time_secs env:production dc:* owner:* frequency:* source:* | sumSeries | summarize 15m | alias \"Total\" | removeEmpty"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "Total HBase puts time taken (Summed 15m interval)",
"tooltip": {
"shared": true,
"sort": 0,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:1153",
"format": "s",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"$$hashKey": "object:1154",
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "M3QL",
"description": "Total number of HBase Clients (Datasets)",
"fieldConfig": {
"defaults": {
"custom": {}
},
"overrides": []
},
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 9,
"w": 12,
"x": 0,
"y": 26
},
"hiddenSeries": false,
"id": 205,
"legend": {
"alignAsTable": true,
"avg": true,
"current": true,
"hideEmpty": true,
"hideZero": true,
"max": true,
"min": true,
"show": true,
"total": false,
"values": true
},
"lines": true,
"linewidth": 2,
"links": [],
"nullPointMode": "connected",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "7.4.2",
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": true,
"steppedLine": false,
"targets": [
{
"hide": false,
"refId": "E",
"target": " fetch service:hoodie datafeed:* name:hbase_num_puts env:production dc:dca* owner:* frequency:* source:* | sumSeries datafeed | count | summarize 15m | removeEmpty"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "Total HBase Clients (Summed 15m interval)",
"tooltip": {
"shared": true,
"sort": 0,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:844",
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"$$hashKey": "object:845",
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "M3QL",
"description": "Total number of HBase Inserts",
"fieldConfig": {
"defaults": {
"custom": {}
},
"overrides": []
},
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 9,
"w": 12,
"x": 12,
"y": 26
},
"hiddenSeries": false,
"id": 206,
"legend": {
"alignAsTable": true,
"avg": true,
"current": true,
"hideEmpty": true,
"hideZero": true,
"max": true,
"min": true,
"show": true,
"total": false,
"values": true
},
"lines": true,
"linewidth": 2,
"links": [],
"nullPointMode": "connected",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "7.4.2",
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": true,
"steppedLine": false,
"targets": [
{
"hide": true,
"refId": "E",
"target": " fetch service:hoodie datafeed:* name:hbase_num_puts env:production dc:dca* owner:* frequency:* source:* | sumSeries | summarize 1h | alias \"Total\" | removeEmpty"
},
{
"hide": false,
"refId": "A",
"target": "fetch service:hoodie datafeed:* name:hbase_qps_resources_actual_puts_time_secs env:production dc:* owner:* frequency:* source:* | sumSeries | summarize 15m | alias \"Total\" | removeEmpty | divideSeries ( fetch service:hoodie datafeed:* name:hbase_num_puts env:production dc:dca* owner:* frequency:* source:* | sumSeries | summarize 15m | alias \"Total\" | removeEmpty) | alias \"Average\""
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "Average HBase Put Time / Key (Summed 15m interval)",
"tooltip": {
"shared": true,
"sort": 0,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:844",
"format": "s",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"$$hashKey": "object:845",
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"aliasColors": {
"Critical 80%": "#BF1B00",
"HDFS usage": "red",
"Warning 75%": "#EAB839"
},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": null,
"description": "HDFS Storage used by the cluster",
"editable": true,
"error": false,
"fieldConfig": {
"defaults": {
"custom": {}
},
"overrides": []
},
"fill": 0,
"fillGradient": 0,
"grid": {},
"gridPos": {
"h": 10,
"w": 12,
"x": 0,
"y": 35
},
"hiddenSeries": false,
"id": 208,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 2,
"links": [],
"nullPointMode": "null",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "7.4.2",
"pointradius": 5,
"points": false,
"renderer": "flot",
"seriesOverrides": [
{
"$$hashKey": "object:2258"
},
{
"$$hashKey": "object:2265",
"alias": "HDFS usage",
"yaxis": 2
}
],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"hide": false,
"refCount": 1,
"refId": "A",
"target": "alias(maxSeries(servers.*.hadoop.{dca1-carbon-prod,magnesium}.hdfs.namenode.CapacityTotalGB), \"Total Capacity\")",
"textEditor": true
},
{
"hide": false,
"refCount": 1,
"refId": "B",
"target": "alias(maxSeries(servers.*.hadoop.{dca1-carbon-prod,magnesium}.hdfs.namenode.CapacityUsedGB), \"Capacity Used\")",
"textEditor": true
},
{
"refCount": 0,
"refId": "C",
"target": "alias(asPercent(#B, #A), 'HDFS usage')",
"targetFull": "alias(asPercent(alias(maxSeries(servers.*.hadoop.{dca1-carbon-prod,magnesium}.hdfs.namenode.CapacityUsedGB), \"Capacity Used\"), alias(maxSeries(servers.*.hadoop.{dca1-carbon-prod,magnesium}.hdfs.namenode.CapacityTotalGB), \"Total Capacity\")), 'HDFS usage')",
"textEditor": true
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "Cluster HDFS Storage Usage (Carbon + Magnesium)",
"tooltip": {
"shared": true,
"sort": 0,
"value_type": "cumulative"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:2126",
"format": "decgbytes",
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"$$hashKey": "object:2127",
"format": "percent",
"logBase": 1,
"max": "100",
"min": "0",
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": null,
"description": "Size of HBase table for the top 25 tables",
"editable": true,
"error": false,
"fieldConfig": {
"defaults": {
"custom": {}
},
"overrides": []
},
"fill": 1,
"fillGradient": 0,
"grid": {},
"gridPos": {
"h": 10,
"w": 12,
"x": 12,
"y": 35
},
"hiddenSeries": false,
"id": 210,
"legend": {
"alignAsTable": true,
"avg": false,
"current": false,
"max": true,
"min": false,
"show": true,
"total": false,
"values": true
},
"lines": true,
"linewidth": 2,
"links": [],
"nullPointMode": "null",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "7.4.2",
"pointradius": 5,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"refId": "A",
"target": "highestMax(groupByNode(servers.hadoop*.hbase.$HbaseClusters.regionserver.tables.*.*.MaxStoreFileSize, 7, 'maxSeries'), 25)",
"textEditor": true
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "HBase Usage (Top 25 tables)",
"tooltip": {
"shared": true,
"sort": 0,
"value_type": "cumulative"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:2396",
"format": "bytes",
"logBase": 1,
"max": null,
"min": 0,
"show": true
},
{
"$$hashKey": "object:2397",
"format": "short",
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": null,
"fieldConfig": {
"defaults": {
"custom": {}
},
"overrides": []
},
"fill": 0,
"fillGradient": 0,
"grid": {},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 45
},
"hiddenSeries": false,
"id": 214,
"legend": {
"avg": false,
"current": true,
"max": false,
"min": false,
"show": true,
"total": false,
"values": true
},
"lines": true,
"linewidth": 1,
"nullPointMode": "connected",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "7.4.2",
"pointradius": 5,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"refCount": 0,
"refId": "A",
"target": "alias(percentileOfSeries(removeAboveValue(nonNegativeDerivative(servers.hadoop*.hbase.$HbaseClusters.regionserver.writeRequestCount), 500000), 95, 'false'), '95 percentile')",
"textEditor": true
},
{
"refCount": 0,
"refId": "B",
"target": "alias(percentileOfSeries(removeAboveValue(nonNegativeDerivative(servers.hadoop*.hbase.$HbaseClusters.regionserver.writeRequestCount), 500000), 50, 'false'), 'median')",
"textEditor": true
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "Write Requests per min",
"tooltip": {
"shared": false,
"sort": 0,
"value_type": "cumulative"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:2868",
"format": "short",
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"$$hashKey": "object:2869",
"format": "short",
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"cacheTimeout": "",
"colorBackground": false,
"colorValue": true,
"colors": [
"rgba(245, 54, 54, 0.9)",
"rgba(237, 129, 40, 0.89)",
"rgba(50, 172, 45, 0.97)"
],
"datasource": null,
"editable": true,
"error": false,
"fieldConfig": {
"defaults": {
"custom": {}
},
"overrides": []
},
"format": "none",
"gauge": {
"maxValue": 100,
"minValue": 0,
"show": false,
"thresholdLabels": false,
"thresholdMarkers": true
},
"gridPos": {
"h": 7,
"w": 5,
"x": 12,
"y": 45
},
"hideTimeOverride": false,
"id": 212,
"interval": null,
"links": [],
"mappingType": 1,
"mappingTypes": [
{
"name": "value to text",
"value": 1
},
{
"name": "range to text",
"value": 2
}
],
"maxDataPoints": "",
"nullPointMode": "connected",
"nullText": null,
"postfix": "",
"postfixFontSize": "50%",
"prefix": "",
"prefixFontSize": "50%",
"rangeMaps": [
{
"from": "null",
"text": "N/A",
"to": "null"
}
],
"sparkline": {
"fillColor": "rgba(31, 118, 189, 0.18)",
"full": false,
"lineColor": "rgb(31, 120, 193)",
"show": true
},
"tableColumn": "",
"targets": [
{
"refId": "A",
"target": "countSeries(averageAbove(servers.hadoop*.hbase.$HbaseClusters.regionserver.MemHeapUsedM, 1))",
"textEditor": true
}
],
"thresholds": "47,49",
"timeFrom": null,
"title": "Total Region Servers",
"transparent": true,
"type": "singlestat",
"valueFontSize": "80%",
"valueMaps": [
{
"op": "=",
"text": "N/A",
"value": "null"
}
],
"valueName": "current",
"y_formats": [
"short",
"short"
]
}
],
"title": "HBase Index Metrics",
"type": "row"
},
{
"collapsed": true,
"datasource": null,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 8
},
"id": 193,
"panels": [
{
"aliasColors": {
"ONCALL_LINE": "#BF1B00"
},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": null,
"editable": true,
"error": false,
"fieldConfig": {
"defaults": {
"custom": {}
},
"overrides": []
},
"fill": 0,
"fillGradient": 0,
"grid": {},
"gridPos": {
"h": 7,
"w": 24,
"x": 0,
"y": 9
},
"hiddenSeries": false,
"id": 88,
"legend": {
"avg": true,
"current": false,
"max": false,
"min": false,
"show": true,
"total": false,
"values": true
},
"lines": true,
"linewidth": 1,
"links": [],
"nullPointMode": "connected",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "7.4.2",
"pointradius": 1,
"points": true,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"refId": "A",
"target": "aliasByNode(highestMax(stats.$dc.gauges.hoodie.$Environment.$Dataset.$Tablefilter.deltastreamer.hiveSyncDuration, 10), 1, 6)",
"textEditor": true
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "Deltastreamer Hive Sync Duration (Top 10)",
"tooltip": {
"shared": false,
"sort": 0,
"value_type": "cumulative"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:1734",
"format": "ms",
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"$$hashKey": "object:1735",
"format": "short",
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"aliasColors": {
"ONCALL_LINE": "#BF1B00"
},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": null,
"editable": true,
"error": false,
"fieldConfig": {
"defaults": {
"custom": {}
},
"overrides": []
},
"fill": 0,
"fillGradient": 0,
"grid": {},
"gridPos": {
"h": 7,
"w": 24,
"x": 0,
"y": 16
},
"hiddenSeries": false,
"id": 89,
"legend": {
"avg": true,
"current": false,
"max": false,
"min": false,
"show": true,
"total": false,
"values": true
},
"lines": true,
"linewidth": 1,
"links": [],
"nullPointMode": "connected",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "7.4.2",
"pointradius": 1,
"points": true,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"refId": "A",
"target": "aliasByNode(highestMax(stats.$dc.gauges.hoodie.$Environment.$Dataset.$Tablefilter.deltastreamer.duration, 10), 1, 6)",
"textEditor": true
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "Transformation + Commit Duration (Top 10)",
"tooltip": {
"shared": false,
"sort": 0,
"value_type": "cumulative"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:1613",
"format": "ms",
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"$$hashKey": "object:1614",
"format": "short",
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
}
],
"title": "Deltastreamer Metrics",
"type": "row"
},
{
"collapsed": true,
"datasource": null,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 9
},
"id": 253,
"panels": [
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": null,
"fieldConfig": {
"defaults": {
"custom": {}
},
"overrides": []
},
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 8,
"w": 8,
"x": 0,
"y": 10
},
"hiddenSeries": false,
"id": 243,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 1,
"nullPointMode": "connected",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "7.4.2",
"pointradius": 1,
"points": true,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"refCount": 0,
"refId": "A",
"target": "aliasByNode(stats.$dc.gauges.hoodie.$Environment.$Tablefilter.replacecommit.duration, 1, 5)",
"textEditor": false
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "Clustering duration",
"tooltip": {
"shared": true,
"sort": 0,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:237",
"format": "ms",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"$$hashKey": "object:238",
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": null,
"fieldConfig": {
"defaults": {
"custom": {}
},
"overrides": []
},
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 8,
"w": 8,
"x": 8,
"y": 10
},
"hiddenSeries": false,
"id": 245,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 1,
"nullPointMode": "connected",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "7.4.2",
"pointradius": 1,
"points": true,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"refId": "A",
"target": "aliasByNode(stats.$dc.gauges.hoodie.$Environment.$Tablefilter.replacecommit.totalPartitionsWritten, 1, 5)",
"textEditor": false
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "Total Partitions Written",
"tooltip": {
"shared": true,
"sort": 0,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:232",
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"$$hashKey": "object:233",
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": null,
"fieldConfig": {
"defaults": {
"custom": {}
},
"overrides": []
},
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 8,
"w": 8,
"x": 16,
"y": 10
},
"hiddenSeries": false,
"id": 260,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 1,
"nullPointMode": "connected",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "7.4.2",
"pointradius": 1,
"points": true,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"refId": "A",
"target": "aliasByNode(stats.$dc.gauges.hoodie.$Environment.$Tablefilter.replacecommit.fileCreationTime, 1, 5)",
"textEditor": false
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "File Creation time",
"tooltip": {
"shared": true,
"sort": 0,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:566",
"format": "ms",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"$$hashKey": "object:567",
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": null,
"fieldConfig": {
"defaults": {
"custom": {}
},
"overrides": []
},
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 8,
"w": 8,
"x": 0,
"y": 18
},
"hiddenSeries": false,
"id": 247,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 1,
"nullPointMode": "connected",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "7.4.2",
"pointradius": 1,
"points": true,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"refId": "A",
"target": "aliasByNode(stats.$dc.gauges.hoodie.$Environment.$Tablefilter.replacecommit.totalFilesInsert, 1, 5)",
"textEditor": false
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "Total Files Inserted",
"tooltip": {
"shared": true,
"sort": 0,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:334",
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"$$hashKey": "object:335",
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": null,
"fieldConfig": {
"defaults": {
"custom": {}
},
"overrides": []
},
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 8,
"w": 8,
"x": 8,
"y": 18
},
"hiddenSeries": false,
"id": 249,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 1,
"nullPointMode": "connected",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "7.4.2",
"pointradius": 1,
"points": true,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"refId": "A",
"target": "aliasByNode(stats.$dc.gauges.hoodie.$Environment.$Tablefilter.replacecommit.totalInsertRecordsWritten, 1, 5)",
"textEditor": false
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "Total Inserted records written",
"tooltip": {
"shared": true,
"sort": 0,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:427",
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"$$hashKey": "object:428",
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"aliasColors": {},
"bars": true,
"dashLength": 10,
"dashes": false,
"datasource": null,
"fieldConfig": {
"defaults": {
"color": {},
"custom": {},
"thresholds": {
"mode": "absolute",
"steps": []
},
"unit": "short"
},
"overrides": []
},
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 8,
"w": 8,
"x": 16,
"y": 18
},
"hiddenSeries": false,
"id": 295,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 1,
"nullPointMode": "null",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "7.4.2",
"pointradius": 2,
"points": true,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"refId": "A",
"target": "stats.$dc.gauges.hoodie.$Environment.$Tablefilter.replacecommit.job.failure",
"textEditor": true
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "Cluster job failures",
"tooltip": {
"shared": true,
"sort": 0,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:103",
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"$$hashKey": "object:104",
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": null,
"fieldConfig": {
"defaults": {
"custom": {}
},
"overrides": []
},
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 8,
"w": 8,
"x": 0,
"y": 26
},
"hiddenSeries": false,
"id": 258,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 1,
"nullPointMode": "null",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "7.4.2",
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"hide": false,
"refCount": 0,
"refId": "A",
"target": "scale(nPercentile(stats.$dc.gauges.hoodie.$Environment.$Tablefilter.replacecommit.encryptRecord, 95), 0.000001)",
"textEditor": false
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "Encryption time taken 95th percentile",
"tooltip": {
"shared": true,
"sort": 0,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:264",
"format": "ms",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"$$hashKey": "object:265",
"format": "Misc",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": null,
"fieldConfig": {
"defaults": {
"custom": {}
},
"overrides": []
},
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 8,
"w": 8,
"x": 8,
"y": 26
},
"hiddenSeries": false,
"id": 281,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 1,
"nullPointMode": "null",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "7.4.2",
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"refId": "A",
"target": "scale(nPercentile(stats.$dc.gauges.hoodie.$Environment.$Tablefilter.replacecommit.totalTransformRecord, 95), 0.000001)"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "Total time taken to transform record 95th percentile",
"tooltip": {
"shared": true,
"sort": 0,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:127",
"format": "ms",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"$$hashKey": "object:128",
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
}
],
"title": "Clustering",
"type": "row"
},
{
"collapsed": true,
"datasource": null,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 10
},
"id": 262,
"panels": [
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": null,
"fieldConfig": {
"defaults": {
"custom": {}
},
"overrides": []
},
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 11
},
"hiddenSeries": false,
"id": 264,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 1,
"nullPointMode": "connected",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "7.4.2",
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"hide": true,
"refCount": 1,
"refId": "A",
"target": "sumSeries(stats.$dc.gauges.hoodie.$Environment.$Dataset.$Tablefilter.DistributedMetrics.*.consolidated.totalRecordsWritten.CREATE)",
"textEditor": true
},
{
"hide": false,
"refCount": 0,
"refId": "B",
"target": "divideSeries(sumSeries(stats.$dc.gauges.hoodie.$Environment.$Dataset.$Tablefilter.DistributedMetrics.*.consolidated.cumulativeParquetWriteTimeInUSec.CREATE), #A)",
"targetFull": "divideSeries(sumSeries(stats.$dc.gauges.hoodie.$Environment.$Dataset.$Tablefilter.DistributedMetrics.*.consolidated.cumulativeParquetWriteTimeInUSec.CREATE), sumSeries(stats.$dc.gauges.hoodie.$Environment.$Dataset.$Tablefilter.DistributedMetrics.*.consolidated.totalRecordsWritten.CREATE))",
"textEditor": true
},
{
"hide": true,
"refCount": 0,
"refId": "C",
"target": ""
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "Per record write time (INSERTS)",
"tooltip": {
"shared": true,
"sort": 0,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:466",
"format": "µs",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"$$hashKey": "object:467",
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": null,
"fieldConfig": {
"defaults": {
"custom": {}
},
"overrides": []
},
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 11
},
"hiddenSeries": false,
"id": 269,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 1,
"nullPointMode": "connected",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "7.4.2",
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"hide": true,
"refCount": 1,
"refId": "A",
"target": "sumSeries(stats.$dc.gauges.hoodie.$Environment.$Dataset.$Tablefilter.DistributedMetrics.*.consolidated.totalRecordsWritten.MERGE)",
"textEditor": true
},
{
"hide": false,
"refCount": 0,
"refId": "B",
"target": "divideSeries(sumSeries(stats.$dc.gauges.hoodie.$Environment.$Dataset.$Tablefilter.DistributedMetrics.*.consolidated.cumulativeParquetWriteTimeInUSec.MERGE), #A)",
"targetFull": "divideSeries(sumSeries(stats.$dc.gauges.hoodie.$Environment.$Dataset.$Tablefilter.DistributedMetrics.*.consolidated.cumulativeParquetWriteTimeInUSec.MERGE), sumSeries(stats.$dc.gauges.hoodie.$Environment.$Dataset.$Tablefilter.DistributedMetrics.*.consolidated.totalRecordsWritten.MERGE))",
"textEditor": true
},
{
"hide": true,
"refCount": 1,
"refId": "C",
"target": ""
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "Per record write time (UPDATES)",
"tooltip": {
"shared": true,
"sort": 0,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:466",
"format": "µs",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"$$hashKey": "object:467",
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": null,
"fieldConfig": {
"defaults": {
"custom": {}
},
"overrides": []
},
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 19
},
"hiddenSeries": false,
"id": 268,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 1,
"nullPointMode": "connected",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "7.4.2",
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"refId": "A",
"target": "sumSeries(stats.$dc.gauges.hoodie.$Environment.$Dataset.$Tablefilter.DistributedMetrics.*.consolidated.cumulativeParquetWriteTimeInUSec.CREATE)",
"textEditor": true
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "Cumulative Write times (INSERTS)",
"tooltip": {
"shared": true,
"sort": 0,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:1067",
"format": "µs",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"$$hashKey": "object:1068",
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": null,
"fieldConfig": {
"defaults": {
"custom": {}
},
"overrides": []
},
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 19
},
"hiddenSeries": false,
"id": 270,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 1,
"nullPointMode": "connected",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "7.4.2",
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"refId": "A",
"target": "sumSeries(stats.$dc.gauges.hoodie.$Environment.$Dataset.$Tablefilter.DistributedMetrics.*.consolidated.cumulativeParquetWriteTimeInUSec.MERGE)",
"textEditor": true
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "Cumulative Write times (UPDATES)",
"tooltip": {
"shared": true,
"sort": 0,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:1067",
"format": "µs",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"$$hashKey": "object:1068",
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": null,
"fieldConfig": {
"defaults": {
"custom": {}
},
"overrides": []
},
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 27
},
"hiddenSeries": false,
"id": 266,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 1,
"nullPointMode": "connected",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "7.4.2",
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"refId": "A",
"target": "sumSeries(stats.$dc.gauges.hoodie.$Environment.$Dataset.$Tablefilter.DistributedMetrics.*.consolidated.totalRecordsWritten.CREATE)",
"textEditor": true
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "Total Records Written (INSERTS)",
"tooltip": {
"shared": true,
"sort": 0,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:802",
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"$$hashKey": "object:803",
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": null,
"fieldConfig": {
"defaults": {
"custom": {}
},
"overrides": []
},
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 27
},
"hiddenSeries": false,
"id": 271,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 1,
"nullPointMode": "connected",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "7.4.2",
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"refId": "A",
"target": "sumSeries(stats.$dc.gauges.hoodie.$Environment.$Dataset.$Tablefilter.DistributedMetrics.*.consolidated.totalRecordsWritten.MERGE)",
"textEditor": true
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "Total Records Written (UPDATES)",
"tooltip": {
"shared": true,
"sort": 0,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:802",
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"$$hashKey": "object:803",
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": null,
"fieldConfig": {
"defaults": {
"custom": {}
},
"overrides": []
},
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 35
},
"hiddenSeries": false,
"id": 273,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": false,
"total": false,
"values": false
},
"lines": true,
"linewidth": 1,
"nullPointMode": "connected",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "7.4.2",
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"hide": false,
"refId": "A",
"target": "highestMax(stats.$dc.gauges.hoodie.$Environment.$Dataset.$Tablefilter.DistributedMetrics.*.writeTimePerRecordInUSec.CREATE.*.*.*.*, 200)",
"textEditor": true
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "Per Record Write Times (INSERT - 200 slowest)",
"tooltip": {
"shared": false,
"sort": 2,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:2074",
"format": "µs",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"$$hashKey": "object:2075",
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "M3QL",
"fieldConfig": {
"defaults": {
"custom": {}
},
"overrides": []
},
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 35
},
"hiddenSeries": false,
"id": 274,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": false,
"total": false,
"values": false
},
"lines": true,
"linewidth": 1,
"nullPointMode": "connected",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "7.4.2",
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"hide": true,
"refId": "A",
"target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.DistributedMetrics.*.writeTimePerRecordInUSec.MERGE.** | sortSeries current desc | head 200"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "Per Record Write Timer (UPDATES - 200 slowest)",
"tooltip": {
"shared": false,
"sort": 2,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:2074",
"format": "µs",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"$$hashKey": "object:2075",
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
}
],
"title": "Write Time Metrics",
"type": "row"
},
{
"collapsed": true,
"datasource": null,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 11
},
"id": 276,
"panels": [
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": null,
"description": "",
"fieldConfig": {
"defaults": {
"custom": {}
},
"overrides": []
},
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 12
},
"hiddenSeries": false,
"id": 278,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 1,
"nullPointMode": "connected",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "7.4.2",
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"refId": "A",
"target": "stats.$dc.gauges.hoodie.$Environment.$Dataset.$Tablefilter.DistributedMetrics.*.consolidated.bloomIndexSearchTime",
"textEditor": true
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "Bloom Index search time",
"tooltip": {
"shared": true,
"sort": 0,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:2459",
"format": "µs",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"$$hashKey": "object:2460",
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": null,
"description": "",
"fieldConfig": {
"defaults": {
"custom": {}
},
"overrides": []
},
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 12
},
"hiddenSeries": false,
"id": 279,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 1,
"nullPointMode": "connected",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "7.4.2",
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"refId": "A",
"target": "stats.$dc.gauges.hoodie.$Environment.$Dataset.$Tablefilter.DistributedMetrics.*.consolidated.bloomIndexFileRecords",
"textEditor": true
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "Bloom Index File Records",
"tooltip": {
"shared": true,
"sort": 0,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:2459",
"format": "µs",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"$$hashKey": "object:2460",
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
}
],
"title": "Bloom Index metrics",
"type": "row"
}
],
"refresh": false,
"schemaVersion": 27,
"style": "dark",
"tags": [],
"templating": {
"list": [
{
"allFormat": "regex values",
"allValue": null,
"current": {
"selected": true,
"text": "production",
"value": "production"
},
"datasource": "M3",
"definition": "",
"description": null,
"error": null,
"hide": 0,
"includeAll": true,
"label": null,
"multi": false,
"name": "Environment",
"options": [
{
"selected": false,
"text": "All",
"value": "$__all"
},
{
"selected": false,
"text": "dev",
"value": "dev"
},
{
"selected": true,
"text": "production",
"value": "production"
},
{
"selected": false,
"text": "staging",
"value": "staging"
},
{
"selected": false,
"text": "prod",
"value": "prod"
}
],
"query": "stats.$dc.gauges.hoodie.*",
"refresh": 0,
"refresh_on_load": false,
"regex": "",
"skipUrlSync": false,
"sort": 0,
"tag": "",
"tagValuesQuery": "",
"tags": [],
"tagsQuery": "",
"type": "query",
"useTags": false
},
{
"allFormat": "wildcard",
"allValue": "*",
"current": {
"selected": false,
"text": "All",
"value": "$__all"
},
"datasource": "M3",
"definition": "",
"description": null,
"error": null,
"hide": 0,
"includeAll": true,
"label": null,
"multi": false,
"name": "Dataset",
"options": [],
"query": "stats.$dc.gauges.hoodie.$Environment.*",
"refresh": 1,
"refresh_on_load": false,
"regex": "",
"skipUrlSync": false,
"sort": 0,
"tag": "",
"tagValuesQuery": "",
"tags": [],
"tagsQuery": "",
"type": "query",
"useTags": false
},
{
"allFormat": "wildcard",
"allValue": "*",
"current": {
"selected": true,
"text": [
"All"
],
"value": [
"$__all"
]
},
"datasource": "M3",
"definition": "",
"description": null,
"error": null,
"hide": 0,
"includeAll": true,
"label": null,
"multi": true,
"name": "Tablefilter",
"options": [],
"query": "stats.$dc.gauges.hoodie.$Environment.$Dataset.*",
"refresh": 1,
"refresh_on_load": false,
"regex": "",
"skipUrlSync": false,
"sort": 0,
"tag": "",
"tagValuesQuery": "",
"tags": [],
"tagsQuery": "",
"type": "query",
"useTags": false
}
]
},
"time": {
"from": "now-1h",
"to": "now"
},
"timepicker": {
"collapse": false,
"enable": true,
"notice": false,
"now": true,
"refresh_intervals": [
"5s",
"10s",
"30s",
"1m",
"5m",
"15m",
"30m",
"1h",
"2h",
"1d"
],
"status": "Stable",
"time_options": [
"5m",
"15m",
"1h",
"6h",
"12h",
"24h",
"2d",
"7d",
"30d"
],
"type": "timepicker"
},
"timezone": "browser",
"title": "Hoodie_OSS",
"uid": "OvPYDsk7k",
"version": 3
}
View Git Blame Copy Permalink