{ "annotations": { "list": [ { "builtIn": 1, "datasource": "-- Grafana --", "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", "name": "Annotations & Alerts", "type": "dashboard" } ] }, "description": "HUDI Dashboard which has been contributed to the OSS project", "editable": true, "gnetId": null, "graphTooltip": 1, "id": 124311, "iteration": 1625260824034, "links": [], "panels": [ { "collapsed": true, "datasource": null, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, "id": 83, "panels": [ { "cacheTimeout": null, "datasource": "M3QL", "fieldConfig": { "defaults": { "custom": {}, "mappings": [ { "id": 0, "op": "=", "text": "N/A", "type": 1, "value": "null" } ], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 6, "w": 3, "x": 0, "y": 1 }, "id": 81, "interval": null, "links": [], "maxDataPoints": 100, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "horizontal", "reduceOptions": { "calcs": [ "mean" ], "fields": "", "values": false }, "text": {}, "textMode": "auto" }, "pluginVersion": "7.4.2", "targets": [ { "refId": "A", "target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.commit.duration | removeEmpty | sum name | count\n" } ], "title": "Dataset Count", "type": "stat" }, { "cacheTimeout": null, "datasource": "M3QL", "fieldConfig": { "defaults": { "custom": {}, "mappings": [ { "id": 0, "op": "=", "text": "N/A", "type": 1, "value": "null" } ], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 6, "w": 3, "x": 3, "y": 1 }, "id": 215, "interval": null, "links": [], "maxDataPoints": 100, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "horizontal", "reduceOptions": { "calcs": [ "mean" ], "fields": "", "values": false }, "text": {}, "textMode": "auto" }, "pluginVersion": "7.4.2", "targets": [ { "refId": "A", "target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.HoodieMetadata.basefile.count | removeEmpty | sumSeries name | count\n" } ], "title": "Datasets with Metadata Enabled", "type": "stat" }, { "cacheTimeout": null, "datasource": "M3QL", "fieldConfig": { "defaults": { "custom": {}, "mappings": [ { "id": 0, "op": "=", "text": "N/A", "type": 1, "value": "null" } ], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 6, "w": 3, "x": 6, "y": 1 }, "id": 147, "interval": null, "links": [], "maxDataPoints": 100, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "horizontal", "reduceOptions": { "calcs": [ "mean" ], "fields": "", "values": false }, "text": {}, "textMode": "auto" }, "pluginVersion": "7.4.2", "targets": [ { "refId": "A", "target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.commit.totalInsertRecordsWritten | sum | moving 1d sum\n" } ], "title": "Records Inserted / Day", "type": "stat" }, { "cacheTimeout": null, "datasource": "M3QL", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "custom": {}, "mappings": [ { "id": 0, "op": "=", "text": "N/A", "type": 1, "value": "null" } ], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 6, "w": 3, "x": 9, "y": 1 }, "id": 148, "interval": null, "links": [], "maxDataPoints": 100, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "horizontal", "reduceOptions": { "calcs": [ "mean" ], "fields": "", "values": false }, "text": {}, "textMode": "auto" }, "pluginVersion": "7.4.2", "targets": [ { "refId": "A", "target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.commit.totalUpdateRecordsWritten | sum | moving 1d sum" } ], "title": "Records Upserted / Day", "type": "stat" }, { "cacheTimeout": null, "datasource": "M3QL", "fieldConfig": { "defaults": { "custom": {}, "mappings": [ { "id": 0, "op": "=", "text": "N/A", "type": 1, "value": "null" } ], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 6, "w": 3, "x": 12, "y": 1 }, "id": 150, "interval": null, "links": [], "maxDataPoints": 100, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "center", "orientation": "horizontal", "reduceOptions": { "calcs": [ "mean" ], "fields": "", "values": false }, "text": {}, "textMode": "value" }, "pluginVersion": "7.4.2", "targets": [ { "refId": "A", "target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.commit.totalRecordsWritten | sum | moving 1d sum" } ], "title": "Records Written / Day", "type": "stat" }, { "cacheTimeout": null, "datasource": null, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "custom": {}, "mappings": [], "thresholds": { "mode": "absolute", "steps": [] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 6, "w": 3, "x": 15, "y": 1 }, "id": 237, "interval": null, "links": [], "maxDataPoints": 100, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "center", "orientation": "horizontal", "reduceOptions": { "calcs": [ "mean" ], "fields": "", "values": false }, "text": {}, "textMode": "value" }, "pluginVersion": "7.4.2", "targets": [ { "refId": "A", "target": "countSeries(groupByNode(stats.$dc.gauges.hoodie.$Environment.$Dataset.$Tablefilter.userName.*, -1, 'sum'))", "textEditor": false } ], "title": "Active Users", "type": "stat" }, { "aliasColors": {}, "breakPoint": "50%", "cacheTimeout": null, "combine": { "label": "Others", "threshold": 0 }, "datasource": "M3QL", "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fontSize": "80%", "format": "short", "gridPos": { "h": 6, "w": 4, "x": 18, "y": 1 }, "id": 236, "interval": null, "legend": { "percentage": false, "show": true, "values": true }, "legendType": "Right side", "links": [], "maxDataPoints": 1, "nullPointMode": "connected", "pieType": "donut", "pluginVersion": "7.4.2", "strokeWidth": 1, "targets": [ { "refId": "A", "target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.version.0.7.0 | sumSeries name | count | alias \"0.7\"" }, { "hide": false, "refId": "B", "target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.finalize.duration | sumSeries name | count | diff (fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.version.0.7.0 | sumSeries name | count) | alias \"0.5\"" } ], "title": "Versions in Production", "type": "grafana-piechart-panel", "valueName": "current" }, { "cacheTimeout": null, "datasource": "M3QL", "description": "Total number of Clean operations / day ", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "custom": {}, "mappings": [], "thresholds": { "mode": "absolute", "steps": [] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 6, "w": 3, "x": 0, "y": 7 }, "id": 240, "interval": null, "links": [], "maxDataPoints": 100, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "center", "orientation": "horizontal", "reduceOptions": { "calcs": [ "mean" ], "fields": "", "values": false }, "text": {}, "textMode": "value" }, "pluginVersion": "7.4.2", "targets": [ { "refId": "A", "target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.clean.duration | removeEmpty | isNonNull | sum | moving 1D sum\n" }, { "hide": true, "refId": "B", "target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.${Tablefilter}.HoodieMetadata.lookup_files.count | removeEmpty | sum | alias \"lookup_files\"" } ], "title": "Clean Operations per / Day", "type": "stat" }, { "cacheTimeout": null, "datasource": "M3QL", "description": "Total number of Commit operations / day ", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "custom": {}, "mappings": [], "thresholds": { "mode": "absolute", "steps": [] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 6, "w": 3, "x": 3, "y": 7 }, "id": 241, "interval": null, "links": [], "maxDataPoints": 100, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "center", "orientation": "horizontal", "reduceOptions": { "calcs": [ "mean" ], "fields": "", "values": false }, "text": {}, "textMode": "value" }, "pluginVersion": "7.4.2", "targets": [ { "refId": "A", "target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.{commit,deltacommit}.duration | removeEmpty | isNonNull | sum | moving 1D sum\n" }, { "hide": true, "refId": "B", "target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.${Tablefilter}.HoodieMetadata.lookup_files.count | removeEmpty | sum | alias \"lookup_files\"" } ], "title": "Commit Operations per / Day", "type": "stat" } ], "title": "Summary", "type": "row" }, { "collapsed": true, "datasource": null, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 1 }, "id": 84, "panels": [ { "aliasColors": { "ONCALL_LINE": "#BF1B00" }, "bars": false, "dashLength": 10, "dashes": false, "datasource": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 0, "fillGradient": 0, "grid": {}, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 2 }, "hiddenSeries": false, "id": 87, "legend": { "avg": true, "current": false, "max": false, "min": false, "show": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "connected", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.4.2", "pointradius": 1, "points": true, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "refId": "A", "target": "aliasByNode(highestMax(stats.$dc.gauges.hoodie.$Environment.$Dataset.$Tablefilter.commit.duration, 10), 1, 6)", "textEditor": false } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Commit Duration (Top 10)", "tooltip": { "shared": false, "sort": 0, "value_type": "cumulative" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:1819", "format": "ms", "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:1820", "format": "short", "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": { "ONCALL_LINE": "#BF1B00" }, "bars": false, "dashLength": 10, "dashes": false, "datasource": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 0, "fillGradient": 0, "grid": {}, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 9 }, "hiddenSeries": false, "id": 80, "legend": { "avg": true, "current": false, "max": false, "min": false, "show": false, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "connected", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.4.2", "pointradius": 1, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "refId": "A", "target": "aliasByNode(nPercentile(stats.$dc.gauges.hoodie.$Environment.$Dataset.$Tablefilter.commit.duration, 95), 5, 6)", "textEditor": true } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Commit Duration (p95)", "tooltip": { "shared": false, "sort": 0, "value_type": "cumulative" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:1903", "format": "ms", "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:1904", "format": "short", "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } } ], "title": "Commit Detailed Metrics", "type": "row" }, { "collapsed": true, "datasource": null, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 2 }, "id": 228, "panels": [ { "aliasColors": { "ONCALL_LINE": "#BF1B00" }, "bars": false, "dashLength": 10, "dashes": false, "datasource": "M3QL", "editable": true, "error": false, "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 0, "fillGradient": 0, "grid": {}, "gridPos": { "h": 9, "w": 12, "x": 0, "y": 3 }, "hiddenSeries": false, "id": 230, "legend": { "avg": true, "current": false, "max": false, "min": false, "show": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "connected", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.4.2", "pointradius": 1, "points": true, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "refId": "A", "target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.clean.duration | sortSeries max desc | head 10\n" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Clean Duration (Top 10)", "tooltip": { "shared": false, "sort": 0, "value_type": "cumulative" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:1819", "format": "ms", "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:1820", "format": "short", "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": { "ONCALL_LINE": "#BF1B00" }, "bars": false, "dashLength": 10, "dashes": false, "datasource": "M3QL", "editable": true, "error": false, "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 0, "fillGradient": 0, "grid": {}, "gridPos": { "h": 9, "w": 12, "x": 12, "y": 3 }, "hiddenSeries": false, "id": 238, "legend": { "avg": true, "current": false, "max": false, "min": false, "show": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "connected", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.4.2", "pointradius": 1, "points": true, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "refId": "A", "target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.clean.numFilesDeleted | sortSeries max desc | head 10\n" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Clean File Deleted", "tooltip": { "shared": false, "sort": 0, "value_type": "cumulative" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:1819", "format": "short", "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:1820", "format": "short", "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": { "ONCALL_LINE": "#BF1B00" }, "bars": false, "dashLength": 10, "dashes": false, "datasource": "M3QL", "editable": true, "error": false, "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 0, "fillGradient": 0, "grid": {}, "gridPos": { "h": 9, "w": 12, "x": 0, "y": 12 }, "hiddenSeries": false, "id": 231, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "connected", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.4.2", "pointradius": 1, "points": true, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "refId": "A", "target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.clean.duration | max | alias \"Maximum\"\n" }, { "hide": false, "refId": "B", "target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.clean.duration | sum | nPercentile 95 | alias \"P95\"\n" }, { "hide": false, "refId": "C", "target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.clean.duration | sum | nPercentile 99 | alias \"P99\"\n" }, { "hide": false, "refId": "D", "target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.clean.duration | sum | moving 1D avg | alias \"AVG\"\n" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Clean Duration (Max, P95, P99)", "tooltip": { "shared": false, "sort": 0, "value_type": "cumulative" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:1819", "format": "ms", "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:1820", "format": "short", "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } } ], "title": "Clean Metrics", "type": "row" }, { "collapsed": true, "datasource": null, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 3 }, "id": 86, "panels": [ { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 1, "fillGradient": 0, "grid": {}, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 4 }, "hiddenSeries": false, "id": 67, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 2, "links": [], "nullPointMode": "connected", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.4.2", "pointradius": 2, "points": true, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "refId": "A", "target": "aliasByNode(highestMax(stats.$dc.gauges.hoodie.$Environment.$Dataset.$Tablefilter.commit.totalFilesInsert.count, 10), 1, 6)", "textEditor": false } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Number of NEW Files Generated (Top 10)", "tooltip": { "shared": false, "sort": 0, "value_type": "cumulative" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:2009", "format": "short", "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:2010", "format": "short", "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 1, "fillGradient": 0, "grid": {}, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 11 }, "hiddenSeries": false, "id": 73, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 2, "links": [], "nullPointMode": "connected", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.4.2", "pointradius": 2, "points": true, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "refId": "A", "target": "aliasByNode(highestMax(stats.$dc.gauges.hoodie.$Environment.$Dataset.$Tablefilter.commit.totalFilesUpdate.count, 10), 1, 6)", "textEditor": true } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Number of EXISTING files versioned (Top 10)", "tooltip": { "shared": false, "sort": 0, "value_type": "cumulative" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:2095", "format": "short", "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:2096", "format": "short", "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 1, "fillGradient": 0, "grid": {}, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 18 }, "hiddenSeries": false, "id": 77, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 2, "links": [], "nullPointMode": "connected", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.4.2", "pointradius": 2, "points": true, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "refId": "A", "target": "aliasByNode(highestMax(stats.$dc.gauges.hoodie.$Environment.$Dataset.$Tablefilter.clean.numFilesDeleted.count, 10), 1, 6)", "textEditor": false } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Number of OLD files deleted (Top 10)", "tooltip": { "shared": false, "sort": 0, "value_type": "cumulative" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:2185", "format": "short", "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:2186", "format": "short", "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 1, "fillGradient": 0, "grid": {}, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 25 }, "hiddenSeries": false, "id": 79, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 2, "links": [], "nullPointMode": "connected", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.4.2", "pointradius": 2, "points": true, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "refId": "A", "target": "aliasByNode(highestMax(stats.$dc.gauges.hoodie.$Environment.$Dataset.$Tablefilter.commit.totalInsertRecordsWritten.count, 10), 1, 6)", "textEditor": true } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Number of NEW records Added (Top 10)", "tooltip": { "shared": false, "sort": 0, "value_type": "cumulative" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:2279", "format": "short", "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:2280", "format": "short", "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 1, "fillGradient": 0, "grid": {}, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 32 }, "hiddenSeries": false, "id": 78, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 2, "links": [], "nullPointMode": "connected", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.4.2", "pointradius": 2, "points": true, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "refId": "A", "target": "aliasByNode(highestMax(stats.$dc.gauges.hoodie.$Environment.$Dataset.$Tablefilter.commit.totalUpdateRecordsWritten.count, 10), 1, 6)", "textEditor": true } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Number of OLD records updated (Top 10)", "tooltip": { "shared": false, "sort": 0, "value_type": "cumulative" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:2365", "format": "short", "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:2366", "format": "short", "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 1, "fillGradient": 0, "grid": {}, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 39 }, "hiddenSeries": false, "id": 75, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 2, "links": [], "nullPointMode": "connected", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.4.2", "pointradius": 2, "points": true, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "refId": "A", "target": "aliasByNode(highestMax(stats.$dc.gauges.hoodie.$Environment.$Dataset.$Tablefilter.commit.totalRecordsWritten.count, 10), 1, 6)", "textEditor": true } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "How many records (INSERT/UPDATE/COPY) written? (Top 10)", "tooltip": { "shared": false, "sort": 0, "value_type": "cumulative" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:2451", "format": "short", "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:2452", "format": "short", "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 1, "fillGradient": 0, "grid": {}, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 46 }, "hiddenSeries": false, "id": 74, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 2, "links": [], "nullPointMode": "connected", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.4.2", "pointradius": 2, "points": true, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "refId": "A", "target": "aliasByNode(highestMax(stats.$dc.gauges.hoodie.$Environment.$Dataset.$Tablefilter.commit.totalPartitionsWritten.count, 10), 1, 6)", "textEditor": false } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "How many partitions updated? (Top 10)", "tooltip": { "shared": false, "sort": 0, "value_type": "cumulative" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:2537", "format": "short", "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:2538", "format": "short", "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 1, "fillGradient": 0, "grid": {}, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 53 }, "hiddenSeries": false, "id": 76, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 2, "links": [], "nullPointMode": "connected", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.4.2", "pointradius": 2, "points": true, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "refId": "A", "target": "aliasByNode(highestMax(stats.$dc.gauges.hoodie.$Environment.$Dataset.$Tablefilter.commit.totalBytesWritten.count, 10), 1, 6)", "textEditor": true } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "How many bytes written? (Top 10)", "tooltip": { "shared": false, "sort": 0, "value_type": "cumulative" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:2623", "format": "bytes", "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:2624", "format": "short", "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } } ], "title": "Dataset Detailed Metrics", "type": "row" }, { "collapsed": true, "datasource": null, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 4 }, "id": 104, "panels": [ { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "M3QL", "description": "Count of various types of HDFS RPC calls for files outside the .hoodie folder", "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 8, "x": 0, "y": 5 }, "hiddenSeries": false, "id": 106, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "connected", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.4.2", "pointradius": 4, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "refId": "A", "target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.HoodieWrapperFileSystem.create | removeEmpty | sumSeries | alias \"create\"\n" }, { "hide": false, "refId": "B", "target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.HoodieWrapperFileSystem.rename | removeEmpty | sumSeries | alias \"rename\"\n" }, { "hide": false, "refId": "C", "target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.HoodieWrapperFileSystem.delete | removeEmpty | sumSeries | alias \"delete\"\n" }, { "hide": false, "refId": "D", "target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.HoodieWrapperFileSystem.listStatus | removeEmpty | sumSeries | alias \"listStatus\"\n" }, { "hide": false, "refId": "E", "target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.HoodieWrapperFileSystem.mkdirs | removeEmpty | sumSeries | alias \"mkdirs\"\n" }, { "hide": false, "refId": "F", "target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.HoodieWrapperFileSystem.getFileStatus | removeEmpty | sumSeries | alias \"getFileStatus\"\n" }, { "hide": false, "refId": "G", "target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.HoodieWrapperFileSystem.globStatus | removeEmpty | sumSeries | alias \"globStatus\"\n" }, { "hide": false, "refId": "H", "target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.HoodieWrapperFileSystem.listFiles | removeEmpty | sumSeries | alias \"listFiles\"\n" }, { "hide": false, "refId": "I", "target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.HoodieWrapperFileSystem.open | removeEmpty | sumSeries | alias \"open\"\n" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "HDFS RPC Calls (outside .hoodie folder)", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:5303", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:5304", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "M3QL", "description": "Count of various types of HDFS RPC calls for files outside the .hoodie folder", "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 8, "x": 8, "y": 5 }, "hiddenSeries": false, "id": 255, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "connected", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.4.2", "pointradius": 4, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "refId": "A", "target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.HoodieWrapperFileSystem.read | removeEmpty | sumSeries | alias \"read\"\n" }, { "hide": false, "refId": "B", "target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.HoodieWrapperFileSystem.write | removeEmpty | sumSeries | alias \"write\"\n" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "HDFS RPC Calls (outside .hoodie folder) - read / write", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:5303", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:5304", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "M3QL", "description": "Total bytes for read/writeHDFS RPC calls", "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 8, "x": 16, "y": 5 }, "hiddenSeries": false, "id": 161, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": true, "values": true }, "lines": true, "linewidth": 1, "nullPointMode": "connected", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.4.2", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "refId": "A", "target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.HoodieWrapperFileSystem.read.totalBytes | removeEmpty | sumSeries | alias \"read\"\n" }, { "hide": false, "refId": "B", "target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.HoodieWrapperFileSystem.write.totalBytes | removeEmpty | sumSeries | alias \"write\"\n" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "HDFS Read / Write Bytes (outside .hoodie folder)", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:5303", "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:5304", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "M3QL", "description": "Count of various types of HDFS RPC calls for files inside the .hoodie folder", "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 8, "x": 0, "y": 13 }, "hiddenSeries": false, "id": 216, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "connected", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.4.2", "pointradius": 4, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "refId": "A", "target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.HoodieWrapperFileSystemMetaFolder.create | removeEmpty | sumSeries | alias \"create\"\n" }, { "hide": false, "refId": "B", "target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.HoodieWrapperFileSystemMetaFolder.rename | removeEmpty | sumSeries | alias \"rename\"\n" }, { "hide": false, "refId": "C", "target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.HoodieWrapperFileSystemMetaFolder.delete | removeEmpty | sumSeries | alias \"delete\"\n" }, { "hide": false, "refId": "D", "target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.HoodieWrapperFileSystemMetaFolder.listStatus | removeEmpty | sumSeries | alias \"listStatus\"\n" }, { "hide": false, "refId": "E", "target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.HoodieWrapperFileSystemMetaFolder.mkdirs | removeEmpty | sumSeries | alias \"mkdirs\"\n" }, { "hide": false, "refId": "F", "target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.HoodieWrapperFileSystemMetaFolder.getFileStatus | removeEmpty | sumSeries | alias \"getFileStatus\"\n" }, { "hide": false, "refId": "G", "target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.HoodieWrapperFileSystemMetaFolder.globStatus | removeEmpty | sumSeries | alias \"globStatus\"\n" }, { "hide": false, "refId": "H", "target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.HoodieWrapperFileSystemMetaFolder.listFiles | removeEmpty | sumSeries | alias \"listFiles\"\n" }, { "hide": false, "refId": "I", "target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.HoodieWrapperFileSystemMetaFolder.open | removeEmpty | sumSeries | alias \"open\"\n" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "HDFS RPC Calls (.hoodie folder)", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:5303", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:5304", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "M3QL", "description": "Count of various types of HDFS RPC calls for files inside the .hoodie folder", "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 8, "x": 8, "y": 13 }, "hiddenSeries": false, "id": 254, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "connected", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.4.2", "pointradius": 4, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "refId": "A", "target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.HoodieWrapperFileSystemMetaFolder.read | removeEmpty | sumSeries | alias \"read\"\n" }, { "hide": false, "refId": "B", "target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.HoodieWrapperFileSystemMetaFolder.write | removeEmpty | sumSeries | alias \"write\"\n" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "HDFS RPC Calls (.hoodie folder) - read / write", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:5303", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:5304", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "M3QL", "description": "Total bytes for read/writeHDFS RPC calls within .hoodie folder", "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 8, "x": 16, "y": 13 }, "hiddenSeries": false, "id": 160, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": true, "values": true }, "lines": true, "linewidth": 1, "nullPointMode": "connected", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.4.2", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "refId": "A", "target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.HoodieWrapperFileSystemMetaFolder.read.totalBytes | removeEmpty | sumSeries | alias \"read\"\n" }, { "hide": false, "refId": "B", "target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.HoodieWrapperFileSystemMetaFolder.write.totalBytes | removeEmpty | sumSeries | alias \"write\"\n" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "HDFS Read / Write Bytes (.hoodie folder)", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:5303", "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:5304", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "M3QL", "description": "Average timings of various types of HDFS RPC calls", "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 8, "x": 0, "y": 21 }, "hiddenSeries": false, "id": 110, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "connected", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.4.2", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "refId": "A", "target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.HoodieWrapperFileSystem*.delete.totalDuration | removeEmpty | sumSeries | divideSeries (fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.HoodieWrapperFileSystem*.delete | removeEmpty | sumSeries) | alias \"delete\"" }, { "hide": false, "refId": "B", "target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.HoodieWrapperFileSystem*.create.totalDuration | removeEmpty | sumSeries | divideSeries (fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.HoodieWrapperFileSystem*.create | removeEmpty | sumSeries) | alias \"create\"" }, { "hide": false, "refId": "C", "target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.HoodieWrapperFileSystem*.rename.totalDuration | removeEmpty | sumSeries | divideSeries (fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.HoodieWrapperFileSystem*.rename | removeEmpty | sumSeries) | alias \"rename\"" }, { "hide": false, "refId": "D", "target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.HoodieWrapperFileSystem*.listStatus.totalDuration | removeEmpty | sumSeries | divideSeries (fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.HoodieWrapperFileSystem*.listStatus | removeEmpty | sumSeries) | alias \"listStatus\"" }, { "hide": false, "refId": "E", "target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.HoodieWrapperFileSystem*.mkdirs.totalDuration | removeEmpty | sumSeries | divideSeries (fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.HoodieWrapperFileSystem*.mkdirs | removeEmpty | sumSeries) | alias \"mkdirs\"" }, { "hide": false, "refId": "F", "target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.HoodieWrapperFileSystem*.getFileStatus.totalDuration | removeEmpty | sumSeries | divideSeries (fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.HoodieWrapperFileSystem*.getFileStatus | removeEmpty | sumSeries) | alias \"getFileStatus\"" }, { "hide": false, "refId": "G", "target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.HoodieWrapperFileSystem*.globStatus.totalDuration | removeEmpty | sumSeries | divideSeries (fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.HoodieWrapperFileSystem*.globStatus | removeEmpty | sumSeries) | alias \"globStatus\"" }, { "hide": false, "refId": "H", "target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.HoodieWrapperFileSystem*.listFiles.totalDuration | removeEmpty | sumSeries | divideSeries (fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.HoodieWrapperFileSystem*.listFiles | removeEmpty | sumSeries) | alias \"listFiles\"" }, { "hide": false, "refId": "I", "target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.HoodieWrapperFileSystem*.read.totalDuration | removeEmpty | sumSeries | divideSeries (fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.HoodieWrapperFileSystem*.read | removeEmpty | sumSeries) | alias \"read\"" }, { "hide": false, "refId": "J", "target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.HoodieWrapperFileSystem*.write.totalDuration | removeEmpty | sumSeries | divideSeries (fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.HoodieWrapperFileSystem*.write | removeEmpty | sumSeries) | alias \"write\"" }, { "hide": false, "refId": "K", "target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.HoodieWrapperFileSystem*.open.totalDuration | removeEmpty | sumSeries | divideSeries (fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.HoodieWrapperFileSystem*.open | removeEmpty | sumSeries) | alias \"open\"" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "HDFS RPC Calls Timings", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:5303", "format": "ms", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:5304", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } } ], "title": "Hadoop & RPC Metrics", "type": "row" }, { "collapsed": true, "datasource": null, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 5 }, "id": 195, "panels": [ { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "M3QL", "description": "Counts of of various types of TimelineService API calls", "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 8, "x": 0, "y": 6 }, "hiddenSeries": false, "id": 177, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "connected", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.4.2", "pointradius": 3, "points": true, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "refId": "A", "target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.TimelineService.ALL_FILEGROUPS_FOR_PARTITION | removeEmpty | sum | alias \"ALL_FILEGROUPS_FOR_PARTITION\"\n" }, { "hide": false, "refId": "B", "target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.TimelineService.ALL_REPLACED_FILEGROUPS_PARTITION | removeEmpty | sum | alias \"ALL_REPLACED_FILEGROUPS_PARTITION\"\n" }, { "hide": false, "refId": "C", "target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.TimelineService.LATEST_DATA_FILES_BEFORE_ON_INSTANT | removeEmpty | sum | alias \"LATEST_DATA_FILES_BEFORE_ON_INSTANT\"\n" }, { "hide": false, "refId": "D", "target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.TimelineService.LATEST_PARTITION_DATA_FILE | removeEmpty | sum | alias \"LATEST_PARTITION_DATA_FILE\"\n" }, { "hide": false, "refId": "E", "target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.TimelineService.PEDING_COMPACTION_OPS | removeEmpty | sum | alias \"PEDING_COMPACTION_OPS\"\n" }, { "hide": false, "refId": "F", "target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.TimelineService.LATEST_ALL_DATA_FILES | removeEmpty | sum | alias \"LATEST_ALL_DATA_FILES\"\n" }, { "hide": false, "refId": "G", "target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.TimelineService.LATEST_DATA_FILE_ON_INSTANT | removeEmpty | sum | alias \"LATEST_DATA_FILE_ON_INSTANT\"\n" }, { "hide": false, "refId": "H", "target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.TimelineService.ALL_DATA_FILES | removeEmpty | sum | alias \"ALL_DATA_FILES\"\n" }, { "hide": false, "refId": "I", "target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.TimelineService.LATEST_DATA_FILES_RANGE_INSTANT | removeEmpty | sum | alias \"LATEST_DATA_FILES_RANGE_INSTANT\"\n" }, { "hide": true, "refId": "J", "target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.TimelineService.* | removeEmpty \n" }, { "hide": false, "refId": "K", "target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.TimelineService.ALL_REPLACED_FILEGROUPS_BEFORE | removeEmpty | sum | alias \"ALL_REPLACED_FILEGROUPS_BEFORE\"\n" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Total API Calls", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:5303", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:5304", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "M3QL", "description": "Timings of various types of TimelineService API calls", "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 8, "x": 8, "y": 6 }, "hiddenSeries": false, "id": 162, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "connected", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.4.2", "pointradius": 3, "points": true, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "refId": "A", "target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.TimelineService.TOTAL_API_TIME | removeEmpty | sumSeries | divideSeries (fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.TimelineService.TOTAL_API_CALLS | removeEmpty | sumSeries) | alias \"API Time\"" }, { "hide": false, "refId": "B", "target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.TimelineService.TOTAL_CHECK_TIME | removeEmpty | sumSeries | divideSeries (fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.TimelineService.TOTAL_API_CALLS | removeEmpty | sumSeries) | alias \"Check Time\"" }, { "hide": false, "refId": "C", "target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.TimelineService.TOTAL_REFRESH_TIME | removeEmpty | sumSeries | divideSeries (fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.TimelineService.TOTAL_API_CALLS | removeEmpty | sumSeries) | alias \"Refresh Time\"" }, { "hide": false, "refId": "D", "target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.TimelineService.TOTAL_HANDLE_TIME | removeEmpty | sumSeries | divideSeries (fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.TimelineService.TOTAL_API_CALLS | removeEmpty | sumSeries) | alias \"Handle Time\"" }, { "hide": false, "refId": "E", "target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.TimelineService.WRITE_VALUE_TIME | removeEmpty | sumSeries | divideSeries (fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.TimelineService.WRITE_VALUE_CNT | removeEmpty | sumSeries) | alias \"Json Encoding Time\"" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Timing for operations (Avg)", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:5303", "format": "ms", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:5304", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "M3QL", "description": "Timings of various types of TimelineService API calls", "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 8, "x": 16, "y": 6 }, "hiddenSeries": false, "id": 219, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "connected", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.4.2", "pointradius": 3, "points": true, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "refId": "A", "target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.TimelineService.TOTAL_API_TIME | removeEmpty | sumSeries | divideSeries (fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.TimelineService.TOTAL_API_CALLS | removeEmpty | sumSeries) | nPercentile 95 | alias \"TimeLine Service P95\"" }, { "hide": false, "refId": "B", "target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.HoodieWrapperFileSystem*.listStatus.totalDuration | removeEmpty | sumSeries | divideSeries (fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.HoodieWrapperFileSystem*.listStatus | removeEmpty | sumSeries) | nPercentile 95 | alias \"HDFS\"" }, { "hide": false, "refId": "C", "target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.TimelineService.TOTAL_API_TIME | removeEmpty | sumSeries | moving 1D sum | divideSeries (fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.TimelineService.TOTAL_API_CALLS | removeEmpty | sumSeries | moving 1D sum) | alias \"TimeLine Service AVG\"" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "TimelineService API vs HDFS RPC Time", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:5303", "format": "ms", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:5304", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } } ], "title": "Timelines Metrics 0.7v", "type": "row" }, { "collapsed": true, "datasource": null, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 6 }, "id": 102, "panels": [ { "cacheTimeout": null, "datasource": "M3QL", "description": "Total number of RPC calls saved / day due to the use of Metadata Table", "fieldConfig": { "defaults": { "custom": {}, "mappings": [ { "id": 0, "op": "=", "text": "N/A", "type": 1, "value": "null" } ], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 6, "w": 4, "x": 0, "y": 7 }, "id": 221, "interval": null, "links": [], "maxDataPoints": 100, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "center", "orientation": "horizontal", "reduceOptions": { "calcs": [ "mean" ], "fields": "", "values": false }, "text": {}, "textMode": "value" }, "pluginVersion": "7.4.2", "targets": [ { "refId": "A", "target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.HoodieWrapperFileSystemMetaFolder.listStatus | sum | moving 1D sum\n" }, { "hide": true, "refId": "B", "target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.${Tablefilter}.HoodieMetadata.lookup_files.count | removeEmpty | sum | alias \"lookup_files\"" } ], "title": "ListStatus Calls Required / Day (.hoodie folder)", "type": "stat" }, { "cacheTimeout": null, "datasource": "M3QL", "description": "Total number of RPC calls saved / day due to the use of Metadata Table", "fieldConfig": { "defaults": { "custom": {}, "mappings": [ { "id": 0, "op": "=", "text": "0", "type": 1, "value": "null" } ], "noValue": "0", "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 6, "w": 4, "x": 4, "y": 7 }, "id": 218, "interval": null, "links": [], "maxDataPoints": 100, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "center", "orientation": "horizontal", "reduceOptions": { "calcs": [ "mean" ], "fields": "", "values": false }, "text": {}, "textMode": "value" }, "pluginVersion": "7.4.2", "targets": [ { "hide": false, "refId": "B", "target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.{HoodieMetadata,HoodieWrapperFileSystem}.{lookup_files.count,lookup_partitions.count,listStatus} | removeEmpty | sum | moving 1D sum\n" } ], "title": "ListStatus Calls Required / Day (outside .hoodie folder)", "type": "stat" }, { "cacheTimeout": null, "datasource": "M3QL", "description": "Total number of RPC calls saved / day due to the use of Metadata Table", "fieldConfig": { "defaults": { "custom": {}, "mappings": [ { "id": 0, "op": "=", "text": "N/A", "type": 1, "value": "null" } ], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 6, "w": 4, "x": 8, "y": 7 }, "id": 224, "interval": null, "links": [], "maxDataPoints": 100, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "center", "orientation": "horizontal", "reduceOptions": { "calcs": [ "mean" ], "fields": "", "values": false }, "text": {}, "textMode": "value" }, "pluginVersion": "7.4.2", "targets": [ { "refId": "A", "target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.{HoodieWrapperFileSystemMetaFolder,HoodieWrapperFileSystem,HoodieMetadata}.{lookup_files.count,lookup_partitions.count,listStatus} | sum | moving 1D sum\n" }, { "hide": true, "refId": "B", "target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.${Tablefilter}.HoodieMetadata.lookup_files.count | removeEmpty | sum | alias \"lookup_files\"" } ], "title": "Total HUDI listStatus Calls / Day", "type": "stat" }, { "cacheTimeout": null, "datasource": "M3QL", "description": "Total number of RPC calls saved / day due to the use of Metadata Table", "fieldConfig": { "defaults": { "custom": {}, "mappings": [ { "id": 0, "op": "=", "text": "0", "type": 1, "value": "null" } ], "noValue": "0", "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 6, "w": 4, "x": 12, "y": 7 }, "id": 222, "interval": null, "links": [], "maxDataPoints": 100, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "center", "orientation": "horizontal", "reduceOptions": { "calcs": [ "mean" ], "fields": "", "values": false }, "text": {}, "textMode": "value" }, "pluginVersion": "7.4.2", "targets": [ { "refId": "A", "target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.HoodieMetadata.{lookup_files,lookup_partitions}.count | sum | moving 1D sum\n" }, { "hide": true, "refId": "B", "target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.${Tablefilter}.HoodieMetadata.lookup_files.count | removeEmpty | sum | alias \"lookup_files\"" } ], "title": "Total listStatus Calls Removed / Day", "type": "stat" }, { "cacheTimeout": null, "datasource": "M3QL", "description": "Percent of listStatus Calls Removed / Day due to the use of Metadata Table", "fieldConfig": { "defaults": { "color": { "mode": "continuous-RdYlGr" }, "custom": {}, "mappings": [], "thresholds": { "mode": "absolute", "steps": [] }, "unit": "percent" }, "overrides": [] }, "gridPos": { "h": 6, "w": 4, "x": 16, "y": 7 }, "id": 223, "interval": null, "links": [], "maxDataPoints": 100, "options": { "orientation": "horizontal", "reduceOptions": { "calcs": [ "mean" ], "fields": "", "values": false }, "showThresholdLabels": true, "showThresholdMarkers": true, "text": {} }, "pluginVersion": "7.4.2", "targets": [ { "hide": false, "refId": "B", "target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.HoodieMetadata.{lookup_files.count,lookup_partitions.count} | sum | moving 1D sum | divideSeries (fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.{HoodieWrapperFileSystemMetaFolder,HoodieWrapperFileSystem,HoodieMetadata}.{lookup_files.count,lookup_partitions.count,listStatus} | removeEmpty | sum | moving 1D sum ) | scale 100\n" } ], "title": "% of listStatus Calls Removed / Day (of Total)", "type": "gauge" }, { "cacheTimeout": null, "datasource": "M3QL", "description": "Percent of listStatus Calls Removed / Day due to the use of Metadata Table as implemented in current production release of HUDI.", "fieldConfig": { "defaults": { "color": { "mode": "continuous-RdYlGr" }, "custom": {}, "mappings": [ { "id": 0, "op": "=", "text": "0", "type": 1, "value": "null" } ], "max": 100, "min": 0, "noValue": "?", "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "percent" }, "overrides": [] }, "gridPos": { "h": 6, "w": 4, "x": 20, "y": 7 }, "id": 226, "interval": null, "links": [], "maxDataPoints": 100, "options": { "orientation": "horizontal", "reduceOptions": { "calcs": [ "mean" ], "fields": "", "values": false }, "showThresholdLabels": true, "showThresholdMarkers": true, "text": {} }, "pluginVersion": "7.4.2", "targets": [ { "refId": "A", "target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.HoodieMetadata.{lookup_files.count,lookup_partitions.count} | sum | divideSeries (fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.{HoodieWrapperFileSystem,HoodieMetadata}.{lookup_files.count,lookup_partitions.count,listStatus} | sum) | scale 100\n" } ], "title": "% of listStatus Calls Removed / Day (As Implemented)", "type": "gauge" }, { "datasource": "M3QL", "description": "Maximum Partitions written in each DeltaCommit (should be 1)", "fieldConfig": { "defaults": { "custom": {}, "mappings": [], "noValue": "0", "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 6, "w": 4, "x": 0, "y": 13 }, "id": 136, "links": [], "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "max" ], "fields": "", "values": false }, "text": {}, "textMode": "auto" }, "pluginVersion": "7.4.2", "targets": [ { "refId": "A", "target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.${Tablefilter}_metadata.deltacommit.totalPartitionsWritten | max\n" } ], "timeFrom": null, "timeShift": null, "title": "DeltaCommit - Max Partitions Written", "type": "stat" }, { "datasource": "M3QL", "description": "Maximum files updated in each DeltaCommit (should be 1)", "fieldConfig": { "defaults": { "custom": {}, "mappings": [], "noValue": "0", "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 6, "w": 4, "x": 4, "y": 13 }, "id": 138, "links": [], "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "max" ], "fields": "", "values": false }, "text": {}, "textMode": "auto" }, "pluginVersion": "7.4.2", "targets": [ { "refId": "A", "target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.${Tablefilter}_metadata.deltacommit.totalFilesUpdate | max\n" } ], "timeFrom": null, "timeShift": null, "title": "DeltaCommit - Max Files Updated", "type": "stat" }, { "datasource": "M3QL", "description": "Maximum records written in a Deltacommit", "fieldConfig": { "defaults": { "custom": {}, "mappings": [], "noValue": "0", "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 6, "w": 4, "x": 8, "y": 13 }, "id": 140, "links": [], "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "max" ], "fields": "", "values": false }, "text": {}, "textMode": "auto" }, "pluginVersion": "7.4.2", "targets": [ { "refId": "A", "target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.${Tablefilter}_metadata.deltacommit.totalRecordsWritten | max\n" } ], "timeFrom": null, "timeShift": null, "title": "DeltaCommit - Max records written", "type": "stat" }, { "datasource": "M3QL", "description": "Maximum Partitions written in each Compaction (should be 1)", "fieldConfig": { "defaults": { "custom": {}, "mappings": [], "noValue": "0", "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 6, "w": 4, "x": 12, "y": 13 }, "id": 142, "links": [], "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "max" ], "fields": "", "values": false }, "text": {}, "textMode": "auto" }, "pluginVersion": "7.4.2", "targets": [ { "refId": "A", "target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.${Tablefilter}_metadata.compaction.totalPartitionsWritten | max\n" } ], "timeFrom": null, "timeShift": null, "title": "Compaction - Max Partitions Written", "type": "stat" }, { "datasource": "M3QL", "description": "Maximum log files compacted in each Compaction (should be 1)", "fieldConfig": { "defaults": { "custom": {}, "mappings": [], "noValue": "0", "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 6, "w": 4, "x": 16, "y": 13 }, "id": 144, "links": [], "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "max" ], "fields": "", "values": false }, "text": {}, "textMode": "auto" }, "pluginVersion": "7.4.2", "targets": [ { "refId": "A", "target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.${Tablefilter}_metadata.compaction.totalLogFilesCompacted | max\n" } ], "timeFrom": null, "timeShift": null, "title": "Compaction - Max LogFiles Compacted", "type": "stat" }, { "datasource": "M3QL", "description": "Maximum records and partitions written in a Compaction", "fieldConfig": { "defaults": { "custom": {}, "mappings": [], "noValue": "0", "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 6, "w": 4, "x": 20, "y": 13 }, "id": 146, "links": [], "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "center", "orientation": "auto", "reduceOptions": { "calcs": [ "max" ], "fields": "", "values": false }, "text": {}, "textMode": "value" }, "pluginVersion": "7.4.2", "targets": [ { "refId": "A", "target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.${Tablefilter}_metadata.compaction.totalRecordsWritten | max\n" } ], "timeFrom": null, "timeShift": null, "title": "Compaction - Max Records Written", "type": "stat" }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "M3QL", "description": "Total time taken for various operations on the metadata table", "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 9, "w": 11, "x": 0, "y": 19 }, "hiddenSeries": false, "id": 112, "legend": { "avg": false, "current": false, "hideEmpty": true, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "connected", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.4.2", "pointradius": 1, "points": true, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "hide": false, "refId": "A", "target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.${Tablefilter}.HoodieMetadata.sync.totalDuration | removeEmpty | sum | divideSeries (fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.${Tablefilter}.HoodieMetadata.sync.count | removeEmpty | sum ) | alias \"sync\"\n" }, { "hide": false, "refId": "B", "target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.${Tablefilter}.HoodieMetadata.scan.totalDuration | removeEmpty | sum | divideSeries (fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.${Tablefilter}.HoodieMetadata.scan.count | removeEmpty | sum ) | alias \"scan\"\n" }, { "hide": false, "refId": "C", "target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.${Tablefilter}.HoodieMetadata.initialize.totalDuration | removeEmpty | sum | divideSeries (fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.${Tablefilter}.HoodieMetadata.initialize.count | removeEmpty | sum ) | alias \"initialize\"\n" }, { "hide": false, "refId": "D", "target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.${Tablefilter}.HoodieMetadata.lookup_partitions.totalDuration | removeEmpty | sum | divideSeries (fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.${Tablefilter}.HoodieMetadata.lookup_partitions.count | removeEmpty | sum ) | alias \"lookup_partitions\"\n" }, { "hide": false, "refId": "E", "target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.${Tablefilter}.HoodieMetadata.lookup_files.totalDuration | removeEmpty | sum | divideSeries (fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.${Tablefilter}.HoodieMetadata.lookup_files.count | removeEmpty | sum ) | alias \"lookup_files\"\n" }, { "hide": false, "refId": "F", "target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.${Tablefilter}.HoodieMetadata.basefile_read.totalDuration | removeEmpty | sum | divideSeries (fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.${Tablefilter}.HoodieMetadata.basefile_read.count | removeEmpty | sum ) | alias \"basefile_read\"\n" }, { "hide": false, "refId": "G", "target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.${Tablefilter}.HoodieMetadata.validate_partitions.totalDuration | removeEmpty | sum | divideSeries (fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.${Tablefilter}.HoodieMetadata.validate_partitions.count | removeEmpty | sum ) | alias \"validate_partitions\"\n" }, { "hide": false, "refId": "H", "target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.${Tablefilter}.HoodieMetadata.validate_files.totalDuration | removeEmpty | sum | divideSeries (fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.${Tablefilter}.HoodieMetadata.validate_files.count | removeEmpty | sum ) | alias \"validate_files\"\n" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Timings for various operations", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:5946", "format": "ms", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:5947", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "M3QL", "description": "Number of lookups for list of partitions or files within a partition from the metadata table", "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 9, "w": 13, "x": 11, "y": 19 }, "hiddenSeries": false, "id": 114, "legend": { "avg": false, "current": false, "hideEmpty": true, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "connected", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.4.2", "pointradius": 1, "points": true, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "refId": "A", "target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.${Tablefilter}.HoodieMetadata.lookup_partitions.count | removeEmpty | sum | alias \"lookup_partitions\"" }, { "hide": false, "refId": "B", "target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.${Tablefilter}.HoodieMetadata.lookup_files.count | removeEmpty | sum | alias \"lookup_files\"" }, { "hide": false, "refId": "C", "target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.${Tablefilter}.HoodieMetadata.sync.count | removeEmpty | sum | alias \"sync\"" }, { "hide": false, "refId": "D", "target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.${Tablefilter}.HoodieMetadata.scan.count | removeEmpty | sum | alias \"scan\"" }, { "hide": false, "refId": "E", "target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.${Tablefilter}.HoodieMetadata.initialize.count | removeEmpty | sum | alias \"initialize\"" }, { "hide": false, "refId": "F", "target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.${Tablefilter}.HoodieMetadata.basefile_read.count | removeEmpty | sum | alias \"basefile_read\"" }, { "hide": false, "refId": "G", "target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.${Tablefilter}.HoodieMetadata.validate_partitions.count | removeEmpty | sum | alias \"validate_partitions\"" }, { "hide": false, "refId": "H", "target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.${Tablefilter}.HoodieMetadata.validate_files.count | removeEmpty | sum | alias \"validate_files\"" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Counts of various operations ", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:5421", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:5422", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "M3QL", "description": "Total size of all metadata base files (HFIle) created", "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 7, "w": 6, "x": 0, "y": 28 }, "hiddenSeries": false, "id": 124, "legend": { "avg": false, "current": false, "hideEmpty": true, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "connected", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.4.2", "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "refId": "A", "target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.${Tablefilter}.HoodieMetadata.basefile.size | removeEmpty | sum name | aliasByTags name\n" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Metadata Basefile Size", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:238", "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:239", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "M3QL", "description": "Number of metadata base files (HFIle) created", "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 7, "w": 7, "x": 6, "y": 28 }, "hiddenSeries": false, "id": 126, "legend": { "avg": false, "current": false, "hideEmpty": true, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 3, "links": [], "nullPointMode": "connected", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.4.2", "pointradius": 0.5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "refId": "A", "target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.${Tablefilter}.HoodieMetadata.basefile.count | removeEmpty | sum name | aliasByTags name\n" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Metadata Basefile Count", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:172", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:173", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "M3QL", "description": "Total size of all metadata log files created", "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 7, "w": 6, "x": 13, "y": 28 }, "hiddenSeries": false, "id": 128, "legend": { "avg": false, "current": false, "hideEmpty": true, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "connected", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.4.2", "pointradius": 0.5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "refId": "A", "target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.${Tablefilter}.HoodieMetadata.logfile.size | removeEmpty | sum name | aliasByTags name\n" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Metadata Logfile Size", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:106", "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:107", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "M3QL", "description": "Number of metadata log files created", "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 7, "w": 5, "x": 19, "y": 28 }, "hiddenSeries": false, "id": 130, "legend": { "avg": false, "current": false, "hideEmpty": true, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "connected", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.4.2", "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "refId": "A", "target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.${Tablefilter}.HoodieMetadata.logfile.count | removeEmpty | sum name | aliasByTags name\n" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Metadata Logfile Count", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:343", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:344", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": { "ONCALL_LINE": "#BF1B00" }, "bars": false, "dashLength": 10, "dashes": false, "datasource": "M3QL", "description": "Maximum Time taken for various actions on the Metadata Table", "editable": true, "error": false, "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 0, "fillGradient": 0, "grid": {}, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 35 }, "hiddenSeries": false, "id": 94, "legend": { "avg": true, "current": false, "max": false, "min": false, "show": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "connected", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.4.2", "pointradius": 1, "points": true, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "refId": "A", "target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.${Tablefilter}_metadata.deltacommit.duration | removeEmpty | avg | alias \"deltacommit\"\n" }, { "hide": false, "refId": "B", "target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.${Tablefilter}_metadata.compaction.duration | removeEmpty | avg | alias \"compaction\"\n" }, { "hide": false, "refId": "C", "target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.${Tablefilter}_metadata.finalize.duration | removeEmpty | avg | alias \"finalize\"\n" }, { "hide": false, "refId": "D", "target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.${Tablefilter}_metadata.clean.duration | removeEmpty | avg | alias \"clean\"\n" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Metadata Table Actions Duration (Average)", "tooltip": { "shared": false, "sort": 0, "value_type": "cumulative" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:2753", "format": "ms", "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:2754", "format": "short", "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": { "ONCALL_LINE": "#BF1B00" }, "bars": false, "dashLength": 10, "dashes": false, "datasource": "M3QL", "description": "Maximum Time taken for various actions on the Metadata Table", "editable": true, "error": false, "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 0, "fillGradient": 0, "grid": {}, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 35 }, "hiddenSeries": false, "id": 217, "legend": { "avg": true, "current": false, "max": false, "min": false, "show": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "connected", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.4.2", "pointradius": 1, "points": true, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "refId": "A", "target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.${Tablefilter}_metadata.deltacommit.duration | removeEmpty | max | alias \"deltacommit\"\n" }, { "hide": false, "refId": "B", "target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.${Tablefilter}_metadata.compaction.duration | removeEmpty | max | alias \"compaction\"\n" }, { "hide": false, "refId": "C", "target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.${Tablefilter}_metadata.finalize.duration | removeEmpty | max | alias \"finalize\"\n" }, { "hide": false, "refId": "D", "target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.${Tablefilter}_metadata.clean.duration | removeEmpty | max | alias \"clean\"\n" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Metadata Table Actions Duration (Max)", "tooltip": { "shared": false, "sort": 0, "value_type": "cumulative" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:2753", "format": "ms", "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:2754", "format": "short", "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "M3QL", "description": "Number of error within the metadata table. ", "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 7, "w": 8, "x": 0, "y": 43 }, "hiddenSeries": false, "id": 118, "legend": { "avg": false, "current": false, "hideEmpty": true, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "connected", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.4.2", "pointradius": 1, "points": true, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "refId": "A", "target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.${Tablefilter}.HoodieMetadata.validate_errors.count | removeEmpty | sum name | alias \"validate_errors\"" }, { "hide": false, "refId": "B", "target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.${Tablefilter}.HoodieMetadata.bootstrap_error.count | removeEmpty | sum name | alias \"bootstrap_errors\"" }, { "hide": false, "refId": "C", "target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.${Tablefilter}.HoodieMetadata.rebootstrap.count | removeEmpty | sum name | alias \"rebootstrap\"" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Metadata Table Errors", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:5760", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:5761", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": { "ONCALL_LINE": "#BF1B00" }, "bars": false, "dashLength": 10, "dashes": false, "datasource": "M3QL", "description": "Records written to Metadata Table in each operation", "editable": true, "error": false, "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 0, "fillGradient": 0, "grid": {}, "gridPos": { "h": 7, "w": 8, "x": 8, "y": 43 }, "hiddenSeries": false, "id": 100, "legend": { "avg": false, "current": false, "max": true, "min": false, "show": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "connected", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.4.2", "pointradius": 1, "points": true, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "refId": "A", "target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.${Tablefilter}_metadata.deltacommit.totalInsertRecordsWritten | removeEmpty | sum | alias \"deltacommit.inserts\"\n" }, { "hide": false, "refId": "B", "target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.${Tablefilter}_metadata.deltacommit.totalUpdateRecordsWritten | removeEmpty | sum | alias \"deltacommit.updates\"\n" }, { "hide": false, "refId": "C", "target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.${Tablefilter}_metadata.deltacommit.totalRecordsWritten | removeEmpty | sum | alias \"deltacommit.total_records\"\n" }, { "hide": false, "refId": "D", "target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.${Tablefilter}_metadata.compaction.totalRecordsWritten | removeEmpty | sum | alias \"compaction.total_records\"\n" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Metadata Table Record Writes", "tooltip": { "shared": false, "sort": 0, "value_type": "cumulative" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:2753", "format": "short", "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:2754", "format": "short", "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } } ], "title": "Metadata Metrics (RFC-15)", "type": "row" }, { "collapsed": true, "datasource": null, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 7 }, "id": 198, "panels": [ { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "M3QL", "description": "Number of HBase Inserts", "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 9, "w": 12, "x": 0, "y": 8 }, "hiddenSeries": false, "id": 200, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": true, "show": true, "total": false, "values": true }, "lines": true, "linewidth": 2, "links": [], "nullPointMode": "connected", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.4.2", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": true, "steppedLine": false, "targets": [ { "hide": true, "refId": "A", "target": " fetch service:hoodie datafeed:* name:hbase_num_puts env:production dc:dca* owner:* frequency:* source:* | sumSeries datafeed | sortSeries current desc | aliasByTags datafeed dc | removeEmpty" }, { "hide": false, "refId": "B", "target": " fetch service:hoodie datafeed:* name:hbase_num_puts env:production dc:dca* owner:* frequency:* source:* | sumSeries datafeed | sortSeries current desc | aliasByTags datafeed dc | removeEmpty | averageSeries | alias \"Average\"" }, { "hide": false, "refId": "C", "target": " fetch service:hoodie datafeed:* name:hbase_num_puts env:production dc:dca* owner:* frequency:* source:* | sumSeries datafeed | sortSeries current desc | aliasByTags datafeed dc | removeEmpty | minSeries | alias \"Min\"" }, { "hide": false, "refId": "D", "target": " fetch service:hoodie datafeed:* name:hbase_num_puts env:production dc:dca* owner:* frequency:* source:* | sumSeries datafeed | sortSeries current desc | aliasByTags datafeed dc | removeEmpty | maxSeries | alias \"Max\"" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Number of HBase Inserts", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:844", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:845", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "M3QL", "description": "Total number of HBase Inserts", "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 9, "w": 12, "x": 12, "y": 8 }, "hiddenSeries": false, "id": 201, "legend": { "alignAsTable": true, "avg": true, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": true, "show": true, "total": false, "values": true }, "lines": true, "linewidth": 2, "links": [], "nullPointMode": "connected", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.4.2", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": true, "steppedLine": false, "targets": [ { "hide": false, "refId": "E", "target": " fetch service:hoodie datafeed:* name:hbase_num_puts env:production dc:dca* owner:* frequency:* source:* | sumSeries | summarize 15m | alias \"Total\" | removeEmpty" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Total HBase Inserts (Summed 15m interval)", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:844", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:845", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "M3QL", "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 9, "w": 12, "x": 0, "y": 17 }, "hiddenSeries": false, "id": 203, "legend": { "alignAsTable": true, "avg": true, "current": true, "hideEmpty": true, "hideZero": true, "max": false, "min": false, "show": true, "total": false, "values": true }, "lines": true, "linewidth": 2, "links": [], "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.4.2", "pointradius": 5, "points": true, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "hide": true, "refId": "A", "target": "fetch service:hoodie datafeed:* name:hbase_qps_resources_actual_puts_time_secs env:production dc:* owner:* frequency:* source:* | sumSeries datafeed | sortSeries current desc | aliasByTags datafeed dc | removeEmpty" }, { "hide": false, "refId": "C", "target": "fetch service:hoodie datafeed:* name:hbase_qps_resources_actual_puts_time_secs env:production dc:* owner:* frequency:* source:* | sumSeries datafeed | sortSeries current desc | aliasByTags datafeed dc | removeEmpty | averageSeries | alias \"Average\"" }, { "hide": false, "refId": "B", "target": "fetch service:hoodie datafeed:* name:hbase_qps_resources_actual_puts_time_secs env:production dc:* owner:* frequency:* source:* | sumSeries datafeed | sortSeries current desc | aliasByTags datafeed dc | removeEmpty | minSeries | alias \"Min\"" }, { "hide": false, "refId": "D", "target": "fetch service:hoodie datafeed:* name:hbase_qps_resources_actual_puts_time_secs env:production dc:* owner:* frequency:* source:* | sumSeries datafeed | sortSeries current desc | aliasByTags datafeed dc | removeEmpty | maxSeries | alias \"Max\"" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "HBase puts time taken", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:1153", "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:1154", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "M3QL", "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 9, "w": 12, "x": 12, "y": 17 }, "hiddenSeries": false, "id": 204, "legend": { "alignAsTable": true, "avg": true, "current": true, "hideEmpty": true, "hideZero": true, "max": false, "min": false, "show": true, "total": false, "values": true }, "lines": true, "linewidth": 2, "links": [], "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.4.2", "pointradius": 5, "points": true, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "hide": false, "refId": "A", "target": "fetch service:hoodie datafeed:* name:hbase_qps_resources_actual_puts_time_secs env:production dc:* owner:* frequency:* source:* | sumSeries | summarize 15m | alias \"Total\" | removeEmpty" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Total HBase puts time taken (Summed 15m interval)", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:1153", "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:1154", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "M3QL", "description": "Total number of HBase Clients (Datasets)", "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 9, "w": 12, "x": 0, "y": 26 }, "hiddenSeries": false, "id": 205, "legend": { "alignAsTable": true, "avg": true, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": true, "show": true, "total": false, "values": true }, "lines": true, "linewidth": 2, "links": [], "nullPointMode": "connected", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.4.2", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": true, "steppedLine": false, "targets": [ { "hide": false, "refId": "E", "target": " fetch service:hoodie datafeed:* name:hbase_num_puts env:production dc:dca* owner:* frequency:* source:* | sumSeries datafeed | count | summarize 15m | removeEmpty" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Total HBase Clients (Summed 15m interval)", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:844", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:845", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "M3QL", "description": "Total number of HBase Inserts", "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 9, "w": 12, "x": 12, "y": 26 }, "hiddenSeries": false, "id": 206, "legend": { "alignAsTable": true, "avg": true, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": true, "show": true, "total": false, "values": true }, "lines": true, "linewidth": 2, "links": [], "nullPointMode": "connected", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.4.2", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": true, "steppedLine": false, "targets": [ { "hide": true, "refId": "E", "target": " fetch service:hoodie datafeed:* name:hbase_num_puts env:production dc:dca* owner:* frequency:* source:* | sumSeries | summarize 1h | alias \"Total\" | removeEmpty" }, { "hide": false, "refId": "A", "target": "fetch service:hoodie datafeed:* name:hbase_qps_resources_actual_puts_time_secs env:production dc:* owner:* frequency:* source:* | sumSeries | summarize 15m | alias \"Total\" | removeEmpty | divideSeries ( fetch service:hoodie datafeed:* name:hbase_num_puts env:production dc:dca* owner:* frequency:* source:* | sumSeries | summarize 15m | alias \"Total\" | removeEmpty) | alias \"Average\"" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Average HBase Put Time / Key (Summed 15m interval)", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:844", "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:845", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": { "Critical 80%": "#BF1B00", "HDFS usage": "red", "Warning 75%": "#EAB839" }, "bars": false, "dashLength": 10, "dashes": false, "datasource": null, "description": "HDFS Storage used by the cluster", "editable": true, "error": false, "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 0, "fillGradient": 0, "grid": {}, "gridPos": { "h": 10, "w": 12, "x": 0, "y": 35 }, "hiddenSeries": false, "id": 208, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 2, "links": [], "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.4.2", "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [ { "$$hashKey": "object:2258" }, { "$$hashKey": "object:2265", "alias": "HDFS usage", "yaxis": 2 } ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "hide": false, "refCount": 1, "refId": "A", "target": "alias(maxSeries(servers.*.hadoop.{dca1-carbon-prod,magnesium}.hdfs.namenode.CapacityTotalGB), \"Total Capacity\")", "textEditor": true }, { "hide": false, "refCount": 1, "refId": "B", "target": "alias(maxSeries(servers.*.hadoop.{dca1-carbon-prod,magnesium}.hdfs.namenode.CapacityUsedGB), \"Capacity Used\")", "textEditor": true }, { "refCount": 0, "refId": "C", "target": "alias(asPercent(#B, #A), 'HDFS usage')", "targetFull": "alias(asPercent(alias(maxSeries(servers.*.hadoop.{dca1-carbon-prod,magnesium}.hdfs.namenode.CapacityUsedGB), \"Capacity Used\"), alias(maxSeries(servers.*.hadoop.{dca1-carbon-prod,magnesium}.hdfs.namenode.CapacityTotalGB), \"Total Capacity\")), 'HDFS usage')", "textEditor": true } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Cluster HDFS Storage Usage (Carbon + Magnesium)", "tooltip": { "shared": true, "sort": 0, "value_type": "cumulative" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:2126", "format": "decgbytes", "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:2127", "format": "percent", "logBase": 1, "max": "100", "min": "0", "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": null, "description": "Size of HBase table for the top 25 tables", "editable": true, "error": false, "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 1, "fillGradient": 0, "grid": {}, "gridPos": { "h": 10, "w": 12, "x": 12, "y": 35 }, "hiddenSeries": false, "id": 210, "legend": { "alignAsTable": true, "avg": false, "current": false, "max": true, "min": false, "show": true, "total": false, "values": true }, "lines": true, "linewidth": 2, "links": [], "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.4.2", "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "refId": "A", "target": "highestMax(groupByNode(servers.hadoop*.hbase.$HbaseClusters.regionserver.tables.*.*.MaxStoreFileSize, 7, 'maxSeries'), 25)", "textEditor": true } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "HBase Usage (Top 25 tables)", "tooltip": { "shared": true, "sort": 0, "value_type": "cumulative" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:2396", "format": "bytes", "logBase": 1, "max": null, "min": 0, "show": true }, { "$$hashKey": "object:2397", "format": "short", "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": null, "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 0, "fillGradient": 0, "grid": {}, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 45 }, "hiddenSeries": false, "id": 214, "legend": { "avg": false, "current": true, "max": false, "min": false, "show": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "nullPointMode": "connected", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.4.2", "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "refCount": 0, "refId": "A", "target": "alias(percentileOfSeries(removeAboveValue(nonNegativeDerivative(servers.hadoop*.hbase.$HbaseClusters.regionserver.writeRequestCount), 500000), 95, 'false'), '95 percentile')", "textEditor": true }, { "refCount": 0, "refId": "B", "target": "alias(percentileOfSeries(removeAboveValue(nonNegativeDerivative(servers.hadoop*.hbase.$HbaseClusters.regionserver.writeRequestCount), 500000), 50, 'false'), 'median')", "textEditor": true } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Write Requests per min", "tooltip": { "shared": false, "sort": 0, "value_type": "cumulative" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:2868", "format": "short", "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:2869", "format": "short", "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "cacheTimeout": "", "colorBackground": false, "colorValue": true, "colors": [ "rgba(245, 54, 54, 0.9)", "rgba(237, 129, 40, 0.89)", "rgba(50, 172, 45, 0.97)" ], "datasource": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "format": "none", "gauge": { "maxValue": 100, "minValue": 0, "show": false, "thresholdLabels": false, "thresholdMarkers": true }, "gridPos": { "h": 7, "w": 5, "x": 12, "y": 45 }, "hideTimeOverride": false, "id": 212, "interval": null, "links": [], "mappingType": 1, "mappingTypes": [ { "name": "value to text", "value": 1 }, { "name": "range to text", "value": 2 } ], "maxDataPoints": "", "nullPointMode": "connected", "nullText": null, "postfix": "", "postfixFontSize": "50%", "prefix": "", "prefixFontSize": "50%", "rangeMaps": [ { "from": "null", "text": "N/A", "to": "null" } ], "sparkline": { "fillColor": "rgba(31, 118, 189, 0.18)", "full": false, "lineColor": "rgb(31, 120, 193)", "show": true }, "tableColumn": "", "targets": [ { "refId": "A", "target": "countSeries(averageAbove(servers.hadoop*.hbase.$HbaseClusters.regionserver.MemHeapUsedM, 1))", "textEditor": true } ], "thresholds": "47,49", "timeFrom": null, "title": "Total Region Servers", "transparent": true, "type": "singlestat", "valueFontSize": "80%", "valueMaps": [ { "op": "=", "text": "N/A", "value": "null" } ], "valueName": "current", "y_formats": [ "short", "short" ] } ], "title": "HBase Index Metrics", "type": "row" }, { "collapsed": true, "datasource": null, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 8 }, "id": 193, "panels": [ { "aliasColors": { "ONCALL_LINE": "#BF1B00" }, "bars": false, "dashLength": 10, "dashes": false, "datasource": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 0, "fillGradient": 0, "grid": {}, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 9 }, "hiddenSeries": false, "id": 88, "legend": { "avg": true, "current": false, "max": false, "min": false, "show": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "connected", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.4.2", "pointradius": 1, "points": true, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "refId": "A", "target": "aliasByNode(highestMax(stats.$dc.gauges.hoodie.$Environment.$Dataset.$Tablefilter.deltastreamer.hiveSyncDuration, 10), 1, 6)", "textEditor": true } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Deltastreamer Hive Sync Duration (Top 10)", "tooltip": { "shared": false, "sort": 0, "value_type": "cumulative" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:1734", "format": "ms", "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:1735", "format": "short", "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": { "ONCALL_LINE": "#BF1B00" }, "bars": false, "dashLength": 10, "dashes": false, "datasource": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 0, "fillGradient": 0, "grid": {}, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 16 }, "hiddenSeries": false, "id": 89, "legend": { "avg": true, "current": false, "max": false, "min": false, "show": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "connected", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.4.2", "pointradius": 1, "points": true, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "refId": "A", "target": "aliasByNode(highestMax(stats.$dc.gauges.hoodie.$Environment.$Dataset.$Tablefilter.deltastreamer.duration, 10), 1, 6)", "textEditor": true } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Transformation + Commit Duration (Top 10)", "tooltip": { "shared": false, "sort": 0, "value_type": "cumulative" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:1613", "format": "ms", "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:1614", "format": "short", "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } } ], "title": "Deltastreamer Metrics", "type": "row" }, { "collapsed": true, "datasource": null, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 9 }, "id": 253, "panels": [ { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": null, "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 8, "x": 0, "y": 10 }, "hiddenSeries": false, "id": 243, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "connected", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.4.2", "pointradius": 1, "points": true, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "refCount": 0, "refId": "A", "target": "aliasByNode(stats.$dc.gauges.hoodie.$Environment.$Tablefilter.replacecommit.duration, 1, 5)", "textEditor": false } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Clustering duration", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:237", "format": "ms", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:238", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": null, "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 8, "x": 8, "y": 10 }, "hiddenSeries": false, "id": 245, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "connected", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.4.2", "pointradius": 1, "points": true, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "refId": "A", "target": "aliasByNode(stats.$dc.gauges.hoodie.$Environment.$Tablefilter.replacecommit.totalPartitionsWritten, 1, 5)", "textEditor": false } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Total Partitions Written", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:232", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:233", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": null, "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 8, "x": 16, "y": 10 }, "hiddenSeries": false, "id": 260, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "connected", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.4.2", "pointradius": 1, "points": true, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "refId": "A", "target": "aliasByNode(stats.$dc.gauges.hoodie.$Environment.$Tablefilter.replacecommit.fileCreationTime, 1, 5)", "textEditor": false } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "File Creation time", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:566", "format": "ms", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:567", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": null, "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 8, "x": 0, "y": 18 }, "hiddenSeries": false, "id": 247, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "connected", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.4.2", "pointradius": 1, "points": true, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "refId": "A", "target": "aliasByNode(stats.$dc.gauges.hoodie.$Environment.$Tablefilter.replacecommit.totalFilesInsert, 1, 5)", "textEditor": false } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Total Files Inserted", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:334", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:335", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": null, "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 8, "x": 8, "y": 18 }, "hiddenSeries": false, "id": 249, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "connected", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.4.2", "pointradius": 1, "points": true, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "refId": "A", "target": "aliasByNode(stats.$dc.gauges.hoodie.$Environment.$Tablefilter.replacecommit.totalInsertRecordsWritten, 1, 5)", "textEditor": false } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Total Inserted records written", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:427", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:428", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": true, "dashLength": 10, "dashes": false, "datasource": null, "fieldConfig": { "defaults": { "color": {}, "custom": {}, "thresholds": { "mode": "absolute", "steps": [] }, "unit": "short" }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 8, "x": 16, "y": 18 }, "hiddenSeries": false, "id": 295, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.4.2", "pointradius": 2, "points": true, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "refId": "A", "target": "stats.$dc.gauges.hoodie.$Environment.$Tablefilter.replacecommit.job.failure", "textEditor": true } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Cluster job failures", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:103", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:104", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": null, "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 8, "x": 0, "y": 26 }, "hiddenSeries": false, "id": 258, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.4.2", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "hide": false, "refCount": 0, "refId": "A", "target": "scale(nPercentile(stats.$dc.gauges.hoodie.$Environment.$Tablefilter.replacecommit.encryptRecord, 95), 0.000001)", "textEditor": false } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Encryption time taken 95th percentile", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:264", "format": "ms", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:265", "format": "Misc", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": null, "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 8, "x": 8, "y": 26 }, "hiddenSeries": false, "id": 281, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.4.2", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "refId": "A", "target": "scale(nPercentile(stats.$dc.gauges.hoodie.$Environment.$Tablefilter.replacecommit.totalTransformRecord, 95), 0.000001)" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Total time taken to transform record 95th percentile", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:127", "format": "ms", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:128", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } } ], "title": "Clustering", "type": "row" }, { "collapsed": true, "datasource": null, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 10 }, "id": 262, "panels": [ { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": null, "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 11 }, "hiddenSeries": false, "id": 264, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "connected", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.4.2", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "hide": true, "refCount": 1, "refId": "A", "target": "sumSeries(stats.$dc.gauges.hoodie.$Environment.$Dataset.$Tablefilter.DistributedMetrics.*.consolidated.totalRecordsWritten.CREATE)", "textEditor": true }, { "hide": false, "refCount": 0, "refId": "B", "target": "divideSeries(sumSeries(stats.$dc.gauges.hoodie.$Environment.$Dataset.$Tablefilter.DistributedMetrics.*.consolidated.cumulativeParquetWriteTimeInUSec.CREATE), #A)", "targetFull": "divideSeries(sumSeries(stats.$dc.gauges.hoodie.$Environment.$Dataset.$Tablefilter.DistributedMetrics.*.consolidated.cumulativeParquetWriteTimeInUSec.CREATE), sumSeries(stats.$dc.gauges.hoodie.$Environment.$Dataset.$Tablefilter.DistributedMetrics.*.consolidated.totalRecordsWritten.CREATE))", "textEditor": true }, { "hide": true, "refCount": 0, "refId": "C", "target": "" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Per record write time (INSERTS)", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:466", "format": "µs", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:467", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": null, "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 11 }, "hiddenSeries": false, "id": 269, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "connected", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.4.2", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "hide": true, "refCount": 1, "refId": "A", "target": "sumSeries(stats.$dc.gauges.hoodie.$Environment.$Dataset.$Tablefilter.DistributedMetrics.*.consolidated.totalRecordsWritten.MERGE)", "textEditor": true }, { "hide": false, "refCount": 0, "refId": "B", "target": "divideSeries(sumSeries(stats.$dc.gauges.hoodie.$Environment.$Dataset.$Tablefilter.DistributedMetrics.*.consolidated.cumulativeParquetWriteTimeInUSec.MERGE), #A)", "targetFull": "divideSeries(sumSeries(stats.$dc.gauges.hoodie.$Environment.$Dataset.$Tablefilter.DistributedMetrics.*.consolidated.cumulativeParquetWriteTimeInUSec.MERGE), sumSeries(stats.$dc.gauges.hoodie.$Environment.$Dataset.$Tablefilter.DistributedMetrics.*.consolidated.totalRecordsWritten.MERGE))", "textEditor": true }, { "hide": true, "refCount": 1, "refId": "C", "target": "" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Per record write time (UPDATES)", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:466", "format": "µs", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:467", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": null, "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 19 }, "hiddenSeries": false, "id": 268, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "connected", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.4.2", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "refId": "A", "target": "sumSeries(stats.$dc.gauges.hoodie.$Environment.$Dataset.$Tablefilter.DistributedMetrics.*.consolidated.cumulativeParquetWriteTimeInUSec.CREATE)", "textEditor": true } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Cumulative Write times (INSERTS)", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:1067", "format": "µs", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:1068", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": null, "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 19 }, "hiddenSeries": false, "id": 270, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "connected", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.4.2", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "refId": "A", "target": "sumSeries(stats.$dc.gauges.hoodie.$Environment.$Dataset.$Tablefilter.DistributedMetrics.*.consolidated.cumulativeParquetWriteTimeInUSec.MERGE)", "textEditor": true } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Cumulative Write times (UPDATES)", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:1067", "format": "µs", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:1068", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": null, "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 27 }, "hiddenSeries": false, "id": 266, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "connected", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.4.2", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "refId": "A", "target": "sumSeries(stats.$dc.gauges.hoodie.$Environment.$Dataset.$Tablefilter.DistributedMetrics.*.consolidated.totalRecordsWritten.CREATE)", "textEditor": true } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Total Records Written (INSERTS)", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:802", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:803", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": null, "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 27 }, "hiddenSeries": false, "id": 271, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "connected", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.4.2", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "refId": "A", "target": "sumSeries(stats.$dc.gauges.hoodie.$Environment.$Dataset.$Tablefilter.DistributedMetrics.*.consolidated.totalRecordsWritten.MERGE)", "textEditor": true } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Total Records Written (UPDATES)", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:802", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:803", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": null, "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 35 }, "hiddenSeries": false, "id": 273, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "connected", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.4.2", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "hide": false, "refId": "A", "target": "highestMax(stats.$dc.gauges.hoodie.$Environment.$Dataset.$Tablefilter.DistributedMetrics.*.writeTimePerRecordInUSec.CREATE.*.*.*.*, 200)", "textEditor": true } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Per Record Write Times (INSERT - 200 slowest)", "tooltip": { "shared": false, "sort": 2, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:2074", "format": "µs", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:2075", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "M3QL", "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 35 }, "hiddenSeries": false, "id": 274, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "connected", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.4.2", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "hide": true, "refId": "A", "target": "fetch service:hoodie type:gauges dc:$dc name:$Environment.$Dataset.$Tablefilter.DistributedMetrics.*.writeTimePerRecordInUSec.MERGE.** | sortSeries current desc | head 200" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Per Record Write Timer (UPDATES - 200 slowest)", "tooltip": { "shared": false, "sort": 2, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:2074", "format": "µs", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:2075", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } } ], "title": "Write Time Metrics", "type": "row" }, { "collapsed": true, "datasource": null, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 11 }, "id": 276, "panels": [ { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": null, "description": "", "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 12 }, "hiddenSeries": false, "id": 278, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "connected", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.4.2", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "refId": "A", "target": "stats.$dc.gauges.hoodie.$Environment.$Dataset.$Tablefilter.DistributedMetrics.*.consolidated.bloomIndexSearchTime", "textEditor": true } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Bloom Index search time", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:2459", "format": "µs", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:2460", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": null, "description": "", "fieldConfig": { "defaults": { "custom": {} }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 12 }, "hiddenSeries": false, "id": 279, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "connected", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.4.2", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "refId": "A", "target": "stats.$dc.gauges.hoodie.$Environment.$Dataset.$Tablefilter.DistributedMetrics.*.consolidated.bloomIndexFileRecords", "textEditor": true } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Bloom Index File Records", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:2459", "format": "µs", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:2460", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } } ], "title": "Bloom Index metrics", "type": "row" } ], "refresh": false, "schemaVersion": 27, "style": "dark", "tags": [], "templating": { "list": [ { "allFormat": "regex values", "allValue": null, "current": { "selected": true, "text": "production", "value": "production" }, "datasource": "M3", "definition": "", "description": null, "error": null, "hide": 0, "includeAll": true, "label": null, "multi": false, "name": "Environment", "options": [ { "selected": false, "text": "All", "value": "$__all" }, { "selected": false, "text": "dev", "value": "dev" }, { "selected": true, "text": "production", "value": "production" }, { "selected": false, "text": "staging", "value": "staging" }, { "selected": false, "text": "prod", "value": "prod" } ], "query": "stats.$dc.gauges.hoodie.*", "refresh": 0, "refresh_on_load": false, "regex": "", "skipUrlSync": false, "sort": 0, "tag": "", "tagValuesQuery": "", "tags": [], "tagsQuery": "", "type": "query", "useTags": false }, { "allFormat": "wildcard", "allValue": "*", "current": { "selected": false, "text": "All", "value": "$__all" }, "datasource": "M3", "definition": "", "description": null, "error": null, "hide": 0, "includeAll": true, "label": null, "multi": false, "name": "Dataset", "options": [], "query": "stats.$dc.gauges.hoodie.$Environment.*", "refresh": 1, "refresh_on_load": false, "regex": "", "skipUrlSync": false, "sort": 0, "tag": "", "tagValuesQuery": "", "tags": [], "tagsQuery": "", "type": "query", "useTags": false }, { "allFormat": "wildcard", "allValue": "*", "current": { "selected": true, "text": [ "All" ], "value": [ "$__all" ] }, "datasource": "M3", "definition": "", "description": null, "error": null, "hide": 0, "includeAll": true, "label": null, "multi": true, "name": "Tablefilter", "options": [], "query": "stats.$dc.gauges.hoodie.$Environment.$Dataset.*", "refresh": 1, "refresh_on_load": false, "regex": "", "skipUrlSync": false, "sort": 0, "tag": "", "tagValuesQuery": "", "tags": [], "tagsQuery": "", "type": "query", "useTags": false } ] }, "time": { "from": "now-1h", "to": "now" }, "timepicker": { "collapse": false, "enable": true, "notice": false, "now": true, "refresh_intervals": [ "5s", "10s", "30s", "1m", "5m", "15m", "30m", "1h", "2h", "1d" ], "status": "Stable", "time_options": [ "5m", "15m", "1h", "6h", "12h", "24h", "2d", "7d", "30d" ], "type": "timepicker" }, "timezone": "browser", "title": "Hoodie_OSS", "uid": "OvPYDsk7k", "version": 3 }