consul/grafana/consul-k8s-control-plane-mo...

{
    "annotations": {
        "list": [
            {
                "builtIn": 1,
                "datasource": {
                    "type": "grafana",
                    "uid": "-- Grafana --"
                },
                "enable": true,
                "hide": true,
                "iconColor": "rgba(0, 211, 255, 1)",
                "name": "Annotations & Alerts",
                "target": {
                    "limit": 100,
                    "matchAny": false,
                    "tags": [],
                    "type": "dashboard"
                },
                "type": "dashboard"
            }
        ]
    },
    "editable": true,
    "fiscalYearStartMonth": 0,
    "graphTooltip": 0,
    "id": 3,
    "links": [],
    "liveNow": false,
    "panels": [
        {
            "collapsed": false,
            "gridPos": {
                "h": 1,
                "w": 24,
                "x": 0,
                "y": 0
            },
            "id": 16,
            "panels": [],
            "title": "Cluster Status",
            "type": "row"
        },
        {
            "datasource": {
                "type": "prometheus",
                "uid": "PBFA97CFB590B2093"
            },
            "fieldConfig": {
                "defaults": {
                    "mappings": [],
                    "thresholds": {
                        "mode": "absolute",
                        "steps": [
                            {
                                "color": "green",
                                "value": null
                            },
                            {
                                "color": "red",
                                "value": 80
                            }
                        ]
                    },
                    "unit": "short"
                },
                "overrides": []
            },
            "gridPos": {
                "h": 8,
                "w": 6,
                "x": 0,
                "y": 1
            },
            "id": 10,
            "options": {
                "colorMode": "value",
                "graphMode": "none",
                "justifyMode": "auto",
                "orientation": "auto",
                "reduceOptions": {
                    "calcs": [
                        "last"
                    ],
                    "fields": "",
                    "limit": 1,
                    "values": true
                },
                "textMode": "auto"
            },
            "pluginVersion": "9.5.5",
            "targets": [
                {
                    "datasource": {
                        "type": "prometheus",
                        "uid": "PBFA97CFB590B2093"
                    },
                    "editorMode": "code",
                    "exemplar": false,
                    "expr": "consul_consul_server_0_consul_members_servers{pod=\"consul-server-0\"}",
                    "hide": false,
                    "instant": true,
                    "legendFormat": "__auto",
                    "range": false,
                    "refId": "A"
                }
            ],
            "title": "Number of Consul Servers",
            "type": "stat"
        },
        {
            "datasource": {
                "type": "prometheus",
                "uid": "PBFA97CFB590B2093"
            },
            "description": "",
            "fieldConfig": {
                "defaults": {
                    "mappings": [],
                    "thresholds": {
                        "mode": "absolute",
                        "steps": [
                            {
                                "color": "green",
                                "value": null
                            },
                            {
                                "color": "red",
                                "value": 80
                            }
                        ]
                    },
                    "unit": "short"
                },
                "overrides": []
            },
            "gridPos": {
                "h": 8,
                "w": 8,
                "x": 6,
                "y": 1
            },
            "id": 60,
            "options": {
                "colorMode": "value",
                "graphMode": "none",
                "justifyMode": "auto",
                "orientation": "auto",
                "reduceOptions": {
                    "calcs": [
                        "last"
                    ],
                    "fields": "",
                    "values": false
                },
                "textMode": "auto"
            },
            "pluginVersion": "9.5.5",
            "targets": [
                {
                    "datasource": {
                        "type": "prometheus",
                        "uid": "PBFA97CFB590B2093"
                    },
                    "editorMode": "builder",
                    "exemplar": false,
                    "expr": "count(consul_dataplane_envoy_connected)",
                    "hide": false,
                    "instant": true,
                    "legendFormat": "__auto",
                    "range": false,
                    "refId": "A"
                }
            ],
            "title": "Number of Connected Consul-dataplanes",
            "type": "stat"
        },
        {
            "datasource": {
                "type": "prometheus",
                "uid": "PBFA97CFB590B2093"
            },
            "description": "Must be 1",
            "fieldConfig": {
                "defaults": {
                    "color": {
                        "mode": "palette-classic"
                    },
                    "custom": {
                        "axisCenteredZero": false,
                        "axisColorMode": "text",
                        "axisLabel": "",
                        "axisPlacement": "auto",
                        "barAlignment": 0,
                        "drawStyle": "line",
                        "fillOpacity": 0,
                        "gradientMode": "none",
                        "hideFrom": {
                            "legend": false,
                            "tooltip": false,
                            "viz": false
                        },
                        "lineInterpolation": "linear",
                        "lineWidth": 1,
                        "pointSize": 5,
                        "scaleDistribution": {
                            "type": "linear"
                        },
                        "showPoints": "auto",
                        "spanNulls": false,
                        "stacking": {
                            "group": "A",
                            "mode": "none"
                        },
                        "thresholdsStyle": {
                            "mode": "off"
                        }
                    },
                    "mappings": [],
                    "thresholds": {
                        "mode": "absolute",
                        "steps": [
                            {
                                "color": "green",
                                "value": null
                            },
                            {
                                "color": "red",
                                "value": 1
                            }
                        ]
                    }
                },
                "overrides": []
            },
            "gridPos": {
                "h": 8,
                "w": 10,
                "x": 14,
                "y": 1
            },
            "id": 12,
            "options": {
                "legend": {
                    "calcs": [],
                    "displayMode": "list",
                    "placement": "bottom",
                    "showLegend": true
                },
                "tooltip": {
                    "mode": "single",
                    "sort": "none"
                }
            },
            "targets": [
                {
                    "datasource": {
                        "type": "prometheus",
                        "uid": "PBFA97CFB590B2093"
                    },
                    "editorMode": "code",
                    "exemplar": false,
                    "expr": "sum({__name__=~\".+server_isLeader\"})",
                    "instant": false,
                    "legendFormat": "__auto",
                    "range": true,
                    "refId": "A"
                }
            ],
            "title": "Number of Leader (1: Healthy)",
            "type": "timeseries"
        },
        {
            "datasource": {
                "type": "prometheus",
                "uid": "PBFA97CFB590B2093"
            },
            "fieldConfig": {
                "defaults": {
                    "mappings": [],
                    "thresholds": {
                        "mode": "absolute",
                        "steps": [
                            {
                                "color": "green",
                                "value": null
                            }
                        ]
                    },
                    "unit": "short"
                },
                "overrides": []
            },
            "gridPos": {
                "h": 8,
                "w": 6,
                "x": 0,
                "y": 9
            },
            "id": 61,
            "options": {
                "colorMode": "value",
                "graphMode": "none",
                "justifyMode": "auto",
                "orientation": "auto",
                "reduceOptions": {
                    "calcs": [
                        "last"
                    ],
                    "fields": "/.*/",
                    "limit": 1,
                    "values": false
                },
                "textMode": "auto"
            },
            "pluginVersion": "9.5.5",
            "targets": [
                {
                    "datasource": {
                        "type": "prometheus",
                        "uid": "PBFA97CFB590B2093"
                    },
                    "editorMode": "code",
                    "exemplar": false,
                    "expr": "max(consul_consul_server_0_version{pod=\"consul-server-0\"}) by (version)",
                    "format": "table",
                    "hide": false,
                    "instant": true,
                    "legendFormat": "",
                    "range": false,
                    "refId": "A"
                }
            ],
            "title": "Consul Server Version",
            "transformations": [
                {
                    "id": "organize",
                    "options": {
                        "excludeByName": {
                            "Time": true,
                            "Value": true
                        },
                        "indexByName": {},
                        "renameByName": {
                            "Time": ""
                        }
                    }
                }
            ],
            "type": "stat"
        },
        {
            "datasource": {
                "type": "prometheus",
                "uid": "PBFA97CFB590B2093"
            },
            "description": "No data in agentless mode",
            "fieldConfig": {
                "defaults": {
                    "mappings": [],
                    "thresholds": {
                        "mode": "absolute",
                        "steps": [
                            {
                                "color": "green",
                                "value": null
                            },
                            {
                                "color": "red",
                                "value": 80
                            }
                        ]
                    },
                    "unit": "short"
                },
                "overrides": []
            },
            "gridPos": {
                "h": 8,
                "w": 8,
                "x": 6,
                "y": 9
            },
            "id": 29,
            "options": {
                "colorMode": "value",
                "graphMode": "none",
                "justifyMode": "auto",
                "orientation": "auto",
                "reduceOptions": {
                    "calcs": [
                        "last"
                    ],
                    "fields": "",
                    "values": false
                },
                "textMode": "auto"
            },
            "pluginVersion": "9.5.5",
            "targets": [
                {
                    "datasource": {
                        "type": "prometheus",
                        "uid": "PBFA97CFB590B2093"
                    },
                    "editorMode": "code",
                    "exemplar": false,
                    "expr": "count(kube_pod_container_resource_limits{pod=~\"consul-client-.*\", container=\"consul\", resource=\"memory\"})",
                    "hide": false,
                    "instant": true,
                    "legendFormat": "__auto",
                    "range": false,
                    "refId": "A"
                }
            ],
            "title": "Number of Consul Clients (No data in agentless mode)",
            "type": "stat"
        },
        {
            "collapsed": true,
            "gridPos": {
                "h": 1,
                "w": 24,
                "x": 0,
                "y": 17
            },
            "id": 40,
            "panels": [
                {
                    "datasource": {
                        "type": "prometheus",
                        "uid": "PBFA97CFB590B2093"
                    },
                    "description": "",
                    "fieldConfig": {
                        "defaults": {
                            "color": {
                                "mode": "palette-classic"
                            },
                            "custom": {
                                "axisCenteredZero": false,
                                "axisColorMode": "text",
                                "axisLabel": "",
                                "axisPlacement": "auto",
                                "barAlignment": 0,
                                "drawStyle": "line",
                                "fillOpacity": 0,
                                "gradientMode": "none",
                                "hideFrom": {
                                    "legend": false,
                                    "tooltip": false,
                                    "viz": false
                                },
                                "lineInterpolation": "linear",
                                "lineWidth": 1,
                                "pointSize": 5,
                                "scaleDistribution": {
                                    "type": "linear"
                                },
                                "showPoints": "auto",
                                "spanNulls": false,
                                "stacking": {
                                    "group": "A",
                                    "mode": "none"
                                },
                                "thresholdsStyle": {
                                    "mode": "off"
                                }
                            },
                            "mappings": [],
                            "thresholds": {
                                "mode": "absolute",
                                "steps": [
                                    {
                                        "color": "green"
                                    },
                                    {
                                        "color": "red",
                                        "value": 80
                                    }
                                ]
                            }
                        },
                        "overrides": []
                    },
                    "gridPos": {
                        "h": 8,
                        "w": 12,
                        "x": 0,
                        "y": 18
                    },
                    "id": 22,
                    "options": {
                        "legend": {
                            "calcs": [],
                            "displayMode": "list",
                            "placement": "bottom",
                            "showLegend": true
                        },
                        "tooltip": {
                            "mode": "single",
                            "sort": "none"
                        }
                    },
                    "targets": [
                        {
                            "datasource": {
                                "type": "prometheus",
                                "uid": "PBFA97CFB590B2093"
                            },
                            "editorMode": "builder",
                            "expr": "consul_raft_leader_lastContact{quantile=\"0.5\"}",
                            "legendFormat": "__auto",
                            "range": true,
                            "refId": "A"
                        },
                        {
                            "datasource": {
                                "type": "prometheus",
                                "uid": "PBFA97CFB590B2093"
                            },
                            "editorMode": "code",
                            "expr": "consul_raft_leader_lastContact{quantile=\"0.99\"}",
                            "hide": false,
                            "legendFormat": "__auto",
                            "range": true,
                            "refId": "B"
                        }
                    ],
                    "title": "Raft Leader LastContact 99th and 50th (ms)",
                    "type": "timeseries"
                },
                {
                    "datasource": {
                        "type": "prometheus",
                        "uid": "PBFA97CFB590B2093"
                    },
                    "description": "Use consul.raft.rpc.appendEntries to understand how long it takes a follower node to process newly received Raft logs from the leader. Like consul.raft.commitTime, increases in this metric can indicate higher load on your Consul servers, and come with a risk of stale data. Since this metric is exposed within each follower, you should aggregate it as both an average (to track overall load on your Raft servers) and a percentile (to watch for outlier nodes).",
                    "fieldConfig": {
                        "defaults": {
                            "color": {
                                "mode": "palette-classic"
                            },
                            "custom": {
                                "axisCenteredZero": false,
                                "axisColorMode": "text",
                                "axisLabel": "",
                                "axisPlacement": "auto",
                                "barAlignment": 0,
                                "drawStyle": "line",
                                "fillOpacity": 0,
                                "gradientMode": "none",
                                "hideFrom": {
                                    "legend": false,
                                    "tooltip": false,
                                    "viz": false
                                },
                                "lineInterpolation": "linear",
                                "lineWidth": 1,
                                "pointSize": 5,
                                "scaleDistribution": {
                                    "type": "linear"
                                },
                                "showPoints": "auto",
                                "spanNulls": false,
                                "stacking": {
                                    "group": "A",
                                    "mode": "none"
                                },
                                "thresholdsStyle": {
                                    "mode": "off"
                                }
                            },
                            "mappings": [],
                            "thresholds": {
                                "mode": "absolute",
                                "steps": [
                                    {
                                        "color": "green"
                                    },
                                    {
                                        "color": "red",
                                        "value": 80
                                    }
                                ]
                            }
                        },
                        "overrides": []
                    },
                    "gridPos": {
                        "h": 8,
                        "w": 12,
                        "x": 12,
                        "y": 18
                    },
                    "id": 18,
                    "options": {
                        "legend": {
                            "calcs": [],
                            "displayMode": "list",
                            "placement": "bottom",
                            "showLegend": true
                        },
                        "tooltip": {
                            "mode": "single",
                            "sort": "none"
                        }
                    },
                    "targets": [
                        {
                            "datasource": {
                                "type": "prometheus",
                                "uid": "PBFA97CFB590B2093"
                            },
                            "editorMode": "code",
                            "expr": "consul_raft_rpc_appendEntries{quantile=\"0.99\"}",
                            "legendFormat": "__auto",
                            "range": true,
                            "refId": "A"
                        },
                        {
                            "datasource": {
                                "type": "prometheus",
                                "uid": "PBFA97CFB590B2093"
                            },
                            "editorMode": "code",
                            "expr": "consul_raft_rpc_appendEntries{quantile=\"0.5\"}",
                            "hide": false,
                            "legendFormat": "__auto",
                            "range": true,
                            "refId": "B"
                        }
                    ],
                    "title": "Follower Append Entries Latency 99th and 50th (ms)",
                    "type": "timeseries"
                },
                {
                    "datasource": {
                        "type": "prometheus",
                        "uid": "PBFA97CFB590B2093"
                    },
                    "fieldConfig": {
                        "defaults": {
                            "color": {
                                "mode": "palette-classic"
                            },
                            "custom": {
                                "axisCenteredZero": false,
                                "axisColorMode": "text",
                                "axisLabel": "",
                                "axisPlacement": "auto",
                                "barAlignment": 0,
                                "drawStyle": "line",
                                "fillOpacity": 0,
                                "gradientMode": "none",
                                "hideFrom": {
                                    "legend": false,
                                    "tooltip": false,
                                    "viz": false
                                },
                                "lineInterpolation": "linear",
                                "lineWidth": 1,
                                "pointSize": 5,
                                "scaleDistribution": {
                                    "type": "linear"
                                },
                                "showPoints": "auto",
                                "spanNulls": false,
                                "stacking": {
                                    "group": "A",
                                    "mode": "none"
                                },
                                "thresholdsStyle": {
                                    "mode": "off"
                                }
                            },
                            "mappings": [],
                            "thresholds": {
                                "mode": "absolute",
                                "steps": [
                                    {
                                        "color": "green"
                                    },
                                    {
                                        "color": "red",
                                        "value": 80
                                    }
                                ]
                            }
                        },
                        "overrides": []
                    },
                    "gridPos": {
                        "h": 8,
                        "w": 12,
                        "x": 0,
                        "y": 26
                    },
                    "id": 20,
                    "options": {
                        "legend": {
                            "calcs": [],
                            "displayMode": "list",
                            "placement": "bottom",
                            "showLegend": true
                        },
                        "tooltip": {
                            "mode": "single",
                            "sort": "none"
                        }
                    },
                    "targets": [
                        {
                            "datasource": {
                                "type": "prometheus",
                                "uid": "PBFA97CFB590B2093"
                            },
                            "editorMode": "code",
                            "expr": "consul_raft_commitTime{quantile=\"0.99\"}",
                            "legendFormat": "__auto",
                            "range": true,
                            "refId": "A"
                        },
                        {
                            "datasource": {
                                "type": "prometheus",
                                "uid": "PBFA97CFB590B2093"
                            },
                            "editorMode": "code",
                            "expr": "consul_raft_commitTime{quantile=\"0.5\"}",
                            "hide": false,
                            "legendFormat": "__auto",
                            "range": true,
                            "refId": "B"
                        }
                    ],
                    "title": "Leader Raft Commit Latency 99th and 50th (ms)",
                    "type": "timeseries"
                },
                {
                    "datasource": {
                        "type": "prometheus",
                        "uid": "PBFA97CFB590B2093"
                    },
                    "fieldConfig": {
                        "defaults": {
                            "color": {
                                "mode": "palette-classic"
                            },
                            "custom": {
                                "axisCenteredZero": false,
                                "axisColorMode": "text",
                                "axisLabel": "",
                                "axisPlacement": "auto",
                                "barAlignment": 0,
                                "drawStyle": "line",
                                "fillOpacity": 0,
                                "gradientMode": "none",
                                "hideFrom": {
                                    "legend": false,
                                    "tooltip": false,
                                    "viz": false
                                },
                                "lineInterpolation": "linear",
                                "lineWidth": 1,
                                "pointSize": 5,
                                "scaleDistribution": {
                                    "type": "linear"
                                },
                                "showPoints": "auto",
                                "spanNulls": false,
                                "stacking": {
                                    "group": "A",
                                    "mode": "none"
                                },
                                "thresholdsStyle": {
                                    "mode": "off"
                                }
                            },
                            "mappings": [],
                            "thresholds": {
                                "mode": "absolute",
                                "steps": [
                                    {
                                        "color": "green"
                                    },
                                    {
                                        "color": "red",
                                        "value": 80
                                    }
                                ]
                            }
                        },
                        "overrides": []
                    },
                    "gridPos": {
                        "h": 8,
                        "w": 12,
                        "x": 12,
                        "y": 26
                    },
                    "id": 46,
                    "options": {
                        "legend": {
                            "calcs": [],
                            "displayMode": "list",
                            "placement": "bottom",
                            "showLegend": true
                        },
                        "tooltip": {
                            "mode": "single",
                            "sort": "none"
                        }
                    },
                    "targets": [
                        {
                            "datasource": {
                                "type": "prometheus",
                                "uid": "PBFA97CFB590B2093"
                            },
                            "editorMode": "code",
                            "expr": "rate(consul_raft_fsm_apply_sum[5m])\n/\nrate(consul_raft_fsm_apply_count[5m])",
                            "legendFormat": "__auto",
                            "range": true,
                            "refId": "A"
                        }
                    ],
                    "title": "Average Raft FSM Apply Latency per 5 minutes (ms)",
                    "type": "timeseries"
                },
                {
                    "datasource": {
                        "type": "prometheus",
                        "uid": "PBFA97CFB590B2093"
                    },
                    "fieldConfig": {
                        "defaults": {
                            "color": {
                                "mode": "palette-classic"
                            },
                            "custom": {
                                "axisCenteredZero": false,
                                "axisColorMode": "text",
                                "axisLabel": "",
                                "axisPlacement": "auto",
                                "barAlignment": 0,
                                "drawStyle": "line",
                                "fillOpacity": 0,
                                "gradientMode": "none",
                                "hideFrom": {
                                    "legend": false,
                                    "tooltip": false,
                                    "viz": false
                                },
                                "lineInterpolation": "linear",
                                "lineWidth": 1,
                                "pointSize": 5,
                                "scaleDistribution": {
                                    "type": "linear"
                                },
                                "showPoints": "auto",
                                "spanNulls": false,
                                "stacking": {
                                    "group": "A",
                                    "mode": "none"
                                },
                                "thresholdsStyle": {
                                    "mode": "off"
                                }
                            },
                            "mappings": [],
                            "thresholds": {
                                "mode": "absolute",
                                "steps": [
                                    {
                                        "color": "green"
                                    },
                                    {
                                        "color": "red",
                                        "value": 80
                                    }
                                ]
                            }
                        },
                        "overrides": []
                    },
                    "gridPos": {
                        "h": 8,
                        "w": 12,
                        "x": 0,
                        "y": 34
                    },
                    "id": 47,
                    "options": {
                        "legend": {
                            "calcs": [],
                            "displayMode": "list",
                            "placement": "bottom",
                            "showLegend": true
                        },
                        "tooltip": {
                            "mode": "single",
                            "sort": "none"
                        }
                    },
                    "targets": [
                        {
                            "datasource": {
                                "type": "prometheus",
                                "uid": "PBFA97CFB590B2093"
                            },
                            "editorMode": "code",
                            "expr": "sum(rate(consul_raft_apply[5m]))",
                            "legendFormat": "__auto",
                            "range": true,
                            "refId": "A"
                        }
                    ],
                    "title": "Raft Apply Rate per 5 minutes",
                    "type": "timeseries"
                },
                {
                    "datasource": {
                        "type": "prometheus",
                        "uid": "PBFA97CFB590B2093"
                    },
                    "description": "An approximate measurement of the proportion of time the main Raft goroutine is busy and unavailable to accept new work.",
                    "fieldConfig": {
                        "defaults": {
                            "color": {
                                "mode": "palette-classic"
                            },
                            "custom": {
                                "axisCenteredZero": false,
                                "axisColorMode": "text",
                                "axisLabel": "",
                                "axisPlacement": "auto",
                                "barAlignment": 0,
                                "drawStyle": "line",
                                "fillOpacity": 0,
                                "gradientMode": "none",
                                "hideFrom": {
                                    "legend": false,
                                    "tooltip": false,
                                    "viz": false
                                },
                                "lineInterpolation": "linear",
                                "lineWidth": 1,
                                "pointSize": 5,
                                "scaleDistribution": {
                                    "type": "linear"
                                },
                                "showPoints": "auto",
                                "spanNulls": false,
                                "stacking": {
                                    "group": "A",
                                    "mode": "none"
                                },
                                "thresholdsStyle": {
                                    "mode": "off"
                                }
                            },
                            "mappings": [],
                            "thresholds": {
                                "mode": "absolute",
                                "steps": [
                                    {
                                        "color": "green"
                                    },
                                    {
                                        "color": "red",
                                        "value": 80
                                    }
                                ]
                            }
                        },
                        "overrides": []
                    },
                    "gridPos": {
                        "h": 8,
                        "w": 12,
                        "x": 12,
                        "y": 34
                    },
                    "id": 41,
                    "options": {
                        "legend": {
                            "calcs": [],
                            "displayMode": "list",
                            "placement": "bottom",
                            "showLegend": true
                        },
                        "tooltip": {
                            "mode": "single",
                            "sort": "none"
                        }
                    },
                    "targets": [
                        {
                            "datasource": {
                                "type": "prometheus",
                                "uid": "PBFA97CFB590B2093"
                            },
                            "editorMode": "builder",
                            "expr": "consul_raft_thread_main_saturation{pod=~\"consul-server-.*\",quantile=\"0.9\"}",
                            "legendFormat": "__auto",
                            "range": true,
                            "refId": "A"
                        }
                    ],
                    "title": "Raft thread main saturation (percentage)",
                    "type": "timeseries"
                },
                {
                    "datasource": {
                        "type": "prometheus",
                        "uid": "PBFA97CFB590B2093"
                    },
                    "description": "An approximate measurement of the proportion of time the FSM Raft goroutine is busy and unavailable to accept new work.",
                    "fieldConfig": {
                        "defaults": {
                            "color": {
                                "mode": "palette-classic"
                            },
                            "custom": {
                                "axisCenteredZero": false,
                                "axisColorMode": "text",
                                "axisLabel": "",
                                "axisPlacement": "auto",
                                "barAlignment": 0,
                                "drawStyle": "line",
                                "fillOpacity": 0,
                                "gradientMode": "none",
                                "hideFrom": {
                                    "legend": false,
                                    "tooltip": false,
                                    "viz": false
                                },
                                "lineInterpolation": "linear",
                                "lineWidth": 1,
                                "pointSize": 5,
                                "scaleDistribution": {
                                    "type": "linear"
                                },
                                "showPoints": "auto",
                                "spanNulls": false,
                                "stacking": {
                                    "group": "A",
                                    "mode": "none"
                                },
                                "thresholdsStyle": {
                                    "mode": "off"
                                }
                            },
                            "mappings": [],
                            "thresholds": {
                                "mode": "absolute",
                                "steps": [
                                    {
                                        "color": "green"
                                    },
                                    {
                                        "color": "red",
                                        "value": 80
                                    }
                                ]
                            }
                        },
                        "overrides": []
                    },
                    "gridPos": {
                        "h": 8,
                        "w": 12,
                        "x": 0,
                        "y": 42
                    },
                    "id": 42,
                    "options": {
                        "legend": {
                            "calcs": [],
                            "displayMode": "list",
                            "placement": "bottom",
                            "showLegend": true
                        },
                        "tooltip": {
                            "mode": "single",
                            "sort": "none"
                        }
                    },
                    "targets": [
                        {
                            "datasource": {
                                "type": "prometheus",
                                "uid": "PBFA97CFB590B2093"
                            },
                            "editorMode": "builder",
                            "expr": "consul_raft_thread_fsm_saturation{pod=~\"consul-server-.*\", quantile=\"0.9\"}",
                            "legendFormat": "__auto",
                            "range": true,
                            "refId": "A"
                        }
                    ],
                    "title": "Raft thread FSM saturation (percentage)",
                    "type": "timeseries"
                }
            ],
            "title": "Raft",
            "type": "row"
        },
        {
            "collapsed": true,
            "gridPos": {
                "h": 1,
                "w": 24,
                "x": 0,
                "y": 18
            },
            "id": 43,
            "panels": [
                {
                    "datasource": {
                        "type": "prometheus",
                        "uid": "PBFA97CFB590B2093"
                    },
                    "description": "Measures the time spent updating the raft store from the serf member information.",
                    "fieldConfig": {
                        "defaults": {
                            "color": {
                                "mode": "palette-classic"
                            },
                            "custom": {
                                "axisCenteredZero": false,
                                "axisColorMode": "text",
                                "axisLabel": "",
                                "axisPlacement": "auto",
                                "barAlignment": 0,
                                "drawStyle": "line",
                                "fillOpacity": 0,
                                "gradientMode": "none",
                                "hideFrom": {
                                    "legend": false,
                                    "tooltip": false,
                                    "viz": false
                                },
                                "lineInterpolation": "linear",
                                "lineWidth": 1,
                                "pointSize": 5,
                                "scaleDistribution": {
                                    "type": "linear"
                                },
                                "showPoints": "auto",
                                "spanNulls": false,
                                "stacking": {
                                    "group": "A",
                                    "mode": "none"
                                },
                                "thresholdsStyle": {
                                    "mode": "off"
                                }
                            },
                            "mappings": [],
                            "thresholds": {
                                "mode": "absolute",
                                "steps": [
                                    {
                                        "color": "green",
                                        "value": null
                                    },
                                    {
                                        "color": "red",
                                        "value": 80
                                    }
                                ]
                            }
                        },
                        "overrides": []
                    },
                    "gridPos": {
                        "h": 8,
                        "w": 12,
                        "x": 0,
                        "y": 19
                    },
                    "id": 44,
                    "options": {
                        "legend": {
                            "calcs": [],
                            "displayMode": "list",
                            "placement": "bottom",
                            "showLegend": true
                        },
                        "tooltip": {
                            "mode": "single",
                            "sort": "none"
                        }
                    },
                    "targets": [
                        {
                            "datasource": {
                                "type": "prometheus",
                                "uid": "PBFA97CFB590B2093"
                            },
                            "editorMode": "code",
                            "expr": "rate(consul_leader_reconcile_sum{pod=~\"consul-server-.*\"}[5m])\n/ \nrate(consul_leader_reconcile_count{pod=~\"consul-server-.*\"}[5m])",
                            "legendFormat": "__auto",
                            "range": true,
                            "refId": "A"
                        }
                    ],
                    "title": "Average latency of leader reconcile per 5 minutes (ms)",
                    "type": "timeseries"
                },
                {
                    "datasource": {
                        "type": "prometheus",
                        "uid": "PBFA97CFB590B2093"
                    },
                    "description": "Increments whenever a Consul server becomes a leader.",
                    "fieldConfig": {
                        "defaults": {
                            "color": {
                                "mode": "palette-classic"
                            },
                            "custom": {
                                "axisCenteredZero": false,
                                "axisColorMode": "text",
                                "axisLabel": "",
                                "axisPlacement": "auto",
                                "barAlignment": 0,
                                "drawStyle": "line",
                                "fillOpacity": 0,
                                "gradientMode": "none",
                                "hideFrom": {
                                    "legend": false,
                                    "tooltip": false,
                                    "viz": false
                                },
                                "lineInterpolation": "linear",
                                "lineWidth": 1,
                                "pointSize": 5,
                                "scaleDistribution": {
                                    "type": "linear"
                                },
                                "showPoints": "auto",
                                "spanNulls": false,
                                "stacking": {
                                    "group": "A",
                                    "mode": "none"
                                },
                                "thresholdsStyle": {
                                    "mode": "off"
                                }
                            },
                            "mappings": [],
                            "thresholds": {
                                "mode": "absolute",
                                "steps": [
                                    {
                                        "color": "green",
                                        "value": null
                                    },
                                    {
                                        "color": "red",
                                        "value": 80
                                    }
                                ]
                            }
                        },
                        "overrides": []
                    },
                    "gridPos": {
                        "h": 8,
                        "w": 12,
                        "x": 12,
                        "y": 19
                    },
                    "id": 45,
                    "options": {
                        "legend": {
                            "calcs": [],
                            "displayMode": "list",
                            "placement": "bottom",
                            "showLegend": true
                        },
                        "tooltip": {
                            "mode": "single",
                            "sort": "none"
                        }
                    },
                    "targets": [
                        {
                            "datasource": {
                                "type": "prometheus",
                                "uid": "PBFA97CFB590B2093"
                            },
                            "editorMode": "code",
                            "expr": "sum(consul_raft_state_leader{pod=~\"consul-server-.*\"})",
                            "legendFormat": "__auto",
                            "range": true,
                            "refId": "A"
                        }
                    ],
                    "title": "Count of a new leader elected [so far] (increasing only)",
                    "type": "timeseries"
                }
            ],
            "title": "Leadership Changes",
            "type": "row"
        },
        {
            "collapsed": true,
            "gridPos": {
                "h": 1,
                "w": 24,
                "x": 0,
                "y": 19
            },
            "id": 14,
            "panels": [
                {
                    "datasource": {
                        "type": "prometheus",
                        "uid": "PBFA97CFB590B2093"
                    },
                    "fieldConfig": {
                        "defaults": {
                            "color": {
                                "mode": "palette-classic"
                            },
                            "custom": {
                                "axisCenteredZero": false,
                                "axisColorMode": "text",
                                "axisLabel": "",
                                "axisPlacement": "auto",
                                "barAlignment": 0,
                                "drawStyle": "line",
                                "fillOpacity": 0,
                                "gradientMode": "none",
                                "hideFrom": {
                                    "legend": false,
                                    "tooltip": false,
                                    "viz": false
                                },
                                "lineInterpolation": "linear",
                                "lineWidth": 1,
                                "pointSize": 5,
                                "scaleDistribution": {
                                    "type": "linear"
                                },
                                "showPoints": "auto",
                                "spanNulls": false,
                                "stacking": {
                                    "group": "A",
                                    "mode": "none"
                                },
                                "thresholdsStyle": {
                                    "mode": "off"
                                }
                            },
                            "mappings": [],
                            "thresholds": {
                                "mode": "absolute",
                                "steps": [
                                    {
                                        "color": "green",
                                        "value": null
                                    },
                                    {
                                        "color": "red",
                                        "value": 80
                                    }
                                ]
                            }
                        },
                        "overrides": []
                    },
                    "gridPos": {
                        "h": 8,
                        "w": 15,
                        "x": 0,
                        "y": 20
                    },
                    "id": 6,
                    "options": {
                        "legend": {
                            "calcs": [],
                            "displayMode": "list",
                            "placement": "bottom",
                            "showLegend": true
                        },
                        "tooltip": {
                            "mode": "single",
                            "sort": "none"
                        }
                    },
                    "targets": [
                        {
                            "datasource": {
                                "type": "prometheus",
                                "uid": "PBFA97CFB590B2093"
                            },
                            "editorMode": "builder",
                            "expr": "rate(container_cpu_usage_seconds_total{container=\"consul\", pod=~\"consul-server-.*\"}[5m])",
                            "hide": false,
                            "legendFormat": "__auto",
                            "range": true,
                            "refId": "A"
                        },
                        {
                            "datasource": {
                                "type": "prometheus",
                                "uid": "PBFA97CFB590B2093"
                            },
                            "editorMode": "builder",
                            "expr": "kube_pod_container_resource_limits{resource=\"cpu\", container=\"consul\"}",
                            "hide": true,
                            "legendFormat": "__auto",
                            "range": true,
                            "refId": "B"
                        }
                    ],
                    "title": "CPU Usage in Seconds (Consul servers)",
                    "type": "timeseries"
                },
                {
                    "datasource": {
                        "type": "prometheus",
                        "uid": "PBFA97CFB590B2093"
                    },
                    "description": "All consul servers have the same limit",
                    "fieldConfig": {
                        "defaults": {
                            "color": {
                                "mode": "thresholds"
                            },
                            "mappings": [],
                            "thresholds": {
                                "mode": "absolute",
                                "steps": [
                                    {
                                        "color": "green",
                                        "value": null
                                    },
                                    {
                                        "color": "red",
                                        "value": 80
                                    }
                                ]
                            }
                        },
                        "overrides": []
                    },
                    "gridPos": {
                        "h": 8,
                        "w": 9,
                        "x": 15,
                        "y": 20
                    },
                    "id": 4,
                    "options": {
                        "orientation": "auto",
                        "reduceOptions": {
                            "calcs": [
                                "lastNotNull"
                            ],
                            "fields": "",
                            "values": false
                        },
                        "showThresholdLabels": false,
                        "showThresholdMarkers": true
                    },
                    "pluginVersion": "9.5.5",
                    "targets": [
                        {
                            "datasource": {
                                "type": "prometheus",
                                "uid": "PBFA97CFB590B2093"
                            },
                            "editorMode": "builder",
                            "exemplar": false,
                            "expr": "kube_pod_container_resource_limits{resource=\"cpu\", pod=\"consul-server-0\"}",
                            "instant": true,
                            "legendFormat": "__auto",
                            "range": false,
                            "refId": "A"
                        }
                    ],
                    "title": "CPU Limit in Seconds (Consul Servers)",
                    "type": "gauge"
                },
                {
                    "datasource": {
                        "type": "prometheus",
                        "uid": "PBFA97CFB590B2093"
                    },
                    "description": "",
                    "fieldConfig": {
                        "defaults": {
                            "color": {
                                "mode": "palette-classic"
                            },
                            "custom": {
                                "axisCenteredZero": false,
                                "axisColorMode": "text",
                                "axisLabel": "",
                                "axisPlacement": "auto",
                                "barAlignment": 0,
                                "drawStyle": "line",
                                "fillOpacity": 0,
                                "gradientMode": "none",
                                "hideFrom": {
                                    "legend": false,
                                    "tooltip": false,
                                    "viz": false
                                },
                                "lineInterpolation": "linear",
                                "lineWidth": 1,
                                "pointSize": 5,
                                "scaleDistribution": {
                                    "type": "linear"
                                },
                                "showPoints": "auto",
                                "spanNulls": false,
                                "stacking": {
                                    "group": "A",
                                    "mode": "none"
                                },
                                "thresholdsStyle": {
                                    "mode": "off"
                                }
                            },
                            "mappings": [],
                            "thresholds": {
                                "mode": "absolute",
                                "steps": [
                                    {
                                        "color": "green",
                                        "value": null
                                    },
                                    {
                                        "color": "red",
                                        "value": 80
                                    }
                                ]
                            },
                            "unit": "bytes"
                        },
                        "overrides": []
                    },
                    "gridPos": {
                        "h": 8,
                        "w": 15,
                        "x": 0,
                        "y": 28
                    },
                    "id": 2,
                    "options": {
                        "legend": {
                            "calcs": [],
                            "displayMode": "list",
                            "placement": "bottom",
                            "showLegend": true
                        },
                        "tooltip": {
                            "mode": "single",
                            "sort": "none"
                        }
                    },
                    "targets": [
                        {
                            "datasource": {
                                "type": "prometheus",
                                "uid": "PBFA97CFB590B2093"
                            },
                            "editorMode": "builder",
                            "expr": "container_memory_working_set_bytes{container=\"consul\", pod=~\"consul-server-.*\"}",
                            "hide": false,
                            "legendFormat": "__auto",
                            "range": true,
                            "refId": "A"
                        }
                    ],
                    "title": "Memory Usage (Consul servers)",
                    "type": "timeseries"
                },
                {
                    "datasource": {
                        "type": "prometheus",
                        "uid": "PBFA97CFB590B2093"
                    },
                    "description": "All consul servers have the same limit",
                    "fieldConfig": {
                        "defaults": {
                            "color": {
                                "mode": "continuous-BlYlRd"
                            },
                            "mappings": [],
                            "thresholds": {
                                "mode": "absolute",
                                "steps": [
                                    {
                                        "color": "green",
                                        "value": null
                                    },
                                    {
                                        "color": "red",
                                        "value": 80
                                    }
                                ]
                            },
                            "unit": "bytes"
                        },
                        "overrides": []
                    },
                    "gridPos": {
                        "h": 8,
                        "w": 9,
                        "x": 15,
                        "y": 28
                    },
                    "id": 8,
                    "options": {
                        "orientation": "auto",
                        "reduceOptions": {
                            "calcs": [
                                "lastNotNull"
                            ],
                            "fields": "",
                            "values": false
                        },
                        "showThresholdLabels": false,
                        "showThresholdMarkers": true
                    },
                    "pluginVersion": "9.5.5",
                    "targets": [
                        {
                            "datasource": {
                                "type": "prometheus",
                                "uid": "PBFA97CFB590B2093"
                            },
                            "editorMode": "code",
                            "exemplar": false,
                            "expr": "kube_pod_container_resource_limits{resource=\"memory\", pod=\"consul-server-0\"}",
                            "instant": true,
                            "legendFormat": "__auto",
                            "range": false,
                            "refId": "A"
                        }
                    ],
                    "title": "Memory Limit (Consul Servers)",
                    "type": "gauge"
                },
                {
                    "datasource": {
                        "type": "prometheus",
                        "uid": "PBFA97CFB590B2093"
                    },
                    "description": "Gives the filesystem storage used by consul servers",
                    "fieldConfig": {
                        "defaults": {
                            "color": {
                                "mode": "continuous-BlYlRd"
                            },
                            "mappings": [],
                            "thresholds": {
                                "mode": "absolute",
                                "steps": [
                                    {
                                        "color": "green",
                                        "value": null
                                    },
                                    {
                                        "color": "red",
                                        "value": 80
                                    }
                                ]
                            },
                            "unit": "bytes"
                        },
                        "overrides": []
                    },
                    "gridPos": {
                        "h": 8,
                        "w": 9,
                        "x": 15,
                        "y": 36
                    },
                    "id": 100,
                    "options": {
                        "orientation": "auto",
                        "reduceOptions": {
                            "calcs": [
                                "lastNotNull"
                            ],
                            "fields": "",
                            "values": false
                        },
                        "showThresholdLabels": false,
                        "showThresholdMarkers": true
                    },
                    "pluginVersion": "9.5.5",
                    "targets": [
                        {
                            "datasource": {
                                "type": "prometheus",
                                "uid": "PBFA97CFB590B2093"
                            },
                            "editorMode": "code",
                            "exemplar": false,
                            "expr": "sum(container_fs_usage_bytes{consul_agent_type=\"server\"}) by (consul_agent_type)",
                            "instant": true,
                            "legendFormat": "{{consul_agent_type}}",
                            "range": false,
                            "refId": "A"
                        }
                    ],
                    "title": "Disk usage (Consul servers)",
                    "type": "gauge"
                },
                {
                    "datasource": {
                        "type": "prometheus",
                        "uid": "PBFA97CFB590B2093"
                    },
                    "description": "",
                    "fieldConfig": {
                        "defaults": {
                            "color": {
                                "mode": "palette-classic"
                            },
                            "custom": {
                                "axisCenteredZero": false,
                                "axisColorMode": "text",
                                "axisLabel": "",
                                "axisPlacement": "auto",
                                "barAlignment": 0,
                                "drawStyle": "line",
                                "fillOpacity": 0,
                                "gradientMode": "none",
                                "hideFrom": {
                                    "legend": false,
                                    "tooltip": false,
                                    "viz": false
                                },
                                "lineInterpolation": "linear",
                                "lineWidth": 1,
                                "pointSize": 5,
                                "scaleDistribution": {
                                    "type": "linear"
                                },
                                "showPoints": "auto",
                                "spanNulls": false,
                                "stacking": {
                                    "group": "A",
                                    "mode": "none"
                                },
                                "thresholdsStyle": {
                                    "mode": "off"
                                }
                            },
                            "mappings": [],
                            "thresholds": {
                                "mode": "absolute",
                                "steps": [
                                    {
                                        "color": "green",
                                        "value": null
                                    },
                                    {
                                        "color": "red",
                                        "value": 80
                                    }
                                ]
                            },
                            "unit": "bytes"
                        },
                        "overrides": []
                    },
                    "gridPos": {
                        "h": 8,
                        "w": 15,
                        "x": 0,
                        "y": 36
                    },
                    "id": 101,
                    "options": {
                        "legend": {
                            "calcs": [],
                            "displayMode": "list",
                            "placement": "bottom",
                            "showLegend": true
                        },
                        "tooltip": {
                            "mode": "single",
                            "sort": "none"
                        }
                    },
                    "targets": [
                        {
                            "datasource": {
                                "type": "prometheus",
                                "uid": "PBFA97CFB590B2093"
                            },
                            "editorMode": "code",
                            "expr": "sum(rate(container_fs_writes_bytes_total{pod=~\"consul-server-.*\", container=\"consul\"}[5m])) by (pod, device)",
                            "hide": false,
                            "legendFormat": "writes, {pod=\"{{pod}}\"}",
                            "range": true,
                            "refId": "A"
                        },
                        {
                            "datasource": {
                                "type": "prometheus",
                                "uid": "PBFA97CFB590B2093"
                            },
                            "editorMode": "code",
                            "expr": "sum(rate(container_fs_reads_bytes_total{pod=~\"consul-server-.*\", container=\"consul\"}[5m])) by (pod, device)",
                            "hide": false,
                            "legendFormat": "reads, {pod=\"{{pod}}\"}",
                            "range": true,
                            "refId": "B"
                        }
                    ],
                    "title": "Disk Read/Write total per 5 minutes (Consul servers)",
                    "type": "timeseries"
                },
                {
                    "datasource": {
                        "type": "prometheus",
                        "uid": "PBFA97CFB590B2093"
                    },
                    "description": "",
                    "fieldConfig": {
                        "defaults": {
                            "color": {
                                "mode": "palette-classic"
                            },
                            "custom": {
                                "axisCenteredZero": false,
                                "axisColorMode": "text",
                                "axisLabel": "",
                                "axisPlacement": "auto",
                                "barAlignment": 0,
                                "drawStyle": "line",
                                "fillOpacity": 0,
                                "gradientMode": "none",
                                "hideFrom": {
                                    "legend": false,
                                    "tooltip": false,
                                    "viz": false
                                },
                                "lineInterpolation": "linear",
                                "lineWidth": 1,
                                "pointSize": 5,
                                "scaleDistribution": {
                                    "type": "linear"
                                },
                                "showPoints": "auto",
                                "spanNulls": false,
                                "stacking": {
                                    "group": "A",
                                    "mode": "none"
                                },
                                "thresholdsStyle": {
                                    "mode": "off"
                                }
                            },
                            "mappings": [],
                            "thresholds": {
                                "mode": "absolute",
                                "steps": [
                                    {
                                        "color": "green",
                                        "value": null
                                    },
                                    {
                                        "color": "red",
                                        "value": 80
                                    }
                                ]
                            },
                            "unit": "bytes"
                        },
                        "overrides": []
                    },
                    "gridPos": {
                        "h": 8,
                        "w": 15,
                        "x": 0,
                        "y": 44
                    },
                    "id": 48,
                    "options": {
                        "legend": {
                            "calcs": [],
                            "displayMode": "list",
                            "placement": "bottom",
                            "showLegend": true
                        },
                        "tooltip": {
                            "mode": "single",
                            "sort": "none"
                        }
                    },
                    "targets": [
                        {
                            "datasource": {
                                "type": "prometheus",
                                "uid": "PBFA97CFB590B2093"
                            },
                            "editorMode": "code",
                            "expr": "sum(rate(container_network_receive_bytes_total{pod=~\"consul-server-.*\"}[5m])) by (pod)",
                            "hide": false,
                            "legendFormat": "__auto",
                            "range": true,
                            "refId": "A"
                        }
                    ],
                    "title": "Received bytes total per 5 minutes (Consul servers)",
                    "type": "timeseries"
                }
            ],
            "title": "Resource Utilization (Consul Servers)",
            "type": "row"
        },
        {
            "collapsed": true,
            "gridPos": {
                "h": 1,
                "w": 24,
                "x": 0,
                "y": 20
            },
            "id": 24,
            "panels": [
                {
                    "datasource": {
                        "type": "prometheus",
                        "uid": "PBFA97CFB590B2093"
                    },
                    "fieldConfig": {
                        "defaults": {
                            "color": {
                                "mode": "palette-classic"
                            },
                            "custom": {
                                "axisCenteredZero": false,
                                "axisColorMode": "text",
                                "axisLabel": "",
                                "axisPlacement": "auto",
                                "barAlignment": 0,
                                "drawStyle": "line",
                                "fillOpacity": 0,
                                "gradientMode": "none",
                                "hideFrom": {
                                    "legend": false,
                                    "tooltip": false,
                                    "viz": false
                                },
                                "lineInterpolation": "linear",
                                "lineWidth": 1,
                                "pointSize": 5,
                                "scaleDistribution": {
                                    "type": "linear"
                                },
                                "showPoints": "auto",
                                "spanNulls": false,
                                "stacking": {
                                    "group": "A",
                                    "mode": "none"
                                },
                                "thresholdsStyle": {
                                    "mode": "off"
                                }
                            },
                            "mappings": [],
                            "thresholds": {
                                "mode": "absolute",
                                "steps": [
                                    {
                                        "color": "green"
                                    },
                                    {
                                        "color": "red",
                                        "value": 80
                                    }
                                ]
                            }
                        },
                        "overrides": []
                    },
                    "gridPos": {
                        "h": 8,
                        "w": 15,
                        "x": 0,
                        "y": 21
                    },
                    "id": 26,
                    "options": {
                        "legend": {
                            "calcs": [],
                            "displayMode": "list",
                            "placement": "bottom",
                            "showLegend": true
                        },
                        "tooltip": {
                            "mode": "single",
                            "sort": "none"
                        }
                    },
                    "targets": [
                        {
                            "datasource": {
                                "type": "prometheus",
                                "uid": "PBFA97CFB590B2093"
                            },
                            "editorMode": "code",
                            "expr": "rate(container_cpu_usage_seconds_total{container=\"consul\", pod=~\"consul-client-.*\"}[5m])",
                            "hide": false,
                            "legendFormat": "__auto",
                            "range": true,
                            "refId": "A"
                        },
                        {
                            "datasource": {
                                "type": "prometheus",
                                "uid": "PBFA97CFB590B2093"
                            },
                            "editorMode": "builder",
                            "expr": "kube_pod_container_resource_limits{resource=\"cpu\", container=\"consul\"}",
                            "hide": true,
                            "legendFormat": "__auto",
                            "range": true,
                            "refId": "B"
                        }
                    ],
                    "title": "CPU Usage in Seconds (Consul Clients)",
                    "type": "timeseries"
                },
                {
                    "datasource": {
                        "type": "prometheus",
                        "uid": "PBFA97CFB590B2093"
                    },
                    "description": "All consul clients have the same limit",
                    "fieldConfig": {
                        "defaults": {
                            "color": {
                                "mode": "thresholds"
                            },
                            "mappings": [],
                            "thresholds": {
                                "mode": "absolute",
                                "steps": [
                                    {
                                        "color": "green"
                                    },
                                    {
                                        "color": "red",
                                        "value": 80
                                    }
                                ]
                            }
                        },
                        "overrides": []
                    },
                    "gridPos": {
                        "h": 8,
                        "w": 9,
                        "x": 15,
                        "y": 21
                    },
                    "id": 28,
                    "options": {
                        "orientation": "auto",
                        "reduceOptions": {
                            "calcs": [
                                "lastNotNull"
                            ],
                            "fields": "",
                            "values": false
                        },
                        "showThresholdLabels": false,
                        "showThresholdMarkers": true
                    },
                    "pluginVersion": "9.5.5",
                    "targets": [
                        {
                            "datasource": {
                                "type": "prometheus",
                                "uid": "PBFA97CFB590B2093"
                            },
                            "editorMode": "code",
                            "exemplar": false,
                            "expr": "max(kube_pod_container_resource_limits{resource=\"cpu\", pod=~\"consul-client-.*\"})",
                            "instant": true,
                            "legendFormat": "__auto",
                            "range": false,
                            "refId": "A"
                        }
                    ],
                    "title": "CPU Limit in Seconds (Consul Clients)",
                    "type": "gauge"
                },
                {
                    "datasource": {
                        "type": "prometheus",
                        "uid": "PBFA97CFB590B2093"
                    },
                    "description": "",
                    "fieldConfig": {
                        "defaults": {
                            "color": {
                                "mode": "palette-classic"
                            },
                            "custom": {
                                "axisCenteredZero": false,
                                "axisColorMode": "text",
                                "axisLabel": "",
                                "axisPlacement": "auto",
                                "barAlignment": 0,
                                "drawStyle": "line",
                                "fillOpacity": 0,
                                "gradientMode": "none",
                                "hideFrom": {
                                    "legend": false,
                                    "tooltip": false,
                                    "viz": false
                                },
                                "lineInterpolation": "linear",
                                "lineWidth": 1,
                                "pointSize": 5,
                                "scaleDistribution": {
                                    "type": "linear"
                                },
                                "showPoints": "auto",
                                "spanNulls": false,
                                "stacking": {
                                    "group": "A",
                                    "mode": "none"
                                },
                                "thresholdsStyle": {
                                    "mode": "off"
                                }
                            },
                            "mappings": [],
                            "thresholds": {
                                "mode": "absolute",
                                "steps": [
                                    {
                                        "color": "green"
                                    },
                                    {
                                        "color": "red",
                                        "value": 80
                                    }
                                ]
                            },
                            "unit": "bytes"
                        },
                        "overrides": []
                    },
                    "gridPos": {
                        "h": 8,
                        "w": 15,
                        "x": 0,
                        "y": 29
                    },
                    "id": 23,
                    "options": {
                        "legend": {
                            "calcs": [],
                            "displayMode": "list",
                            "placement": "bottom",
                            "showLegend": true
                        },
                        "tooltip": {
                            "mode": "single",
                            "sort": "none"
                        }
                    },
                    "targets": [
                        {
                            "datasource": {
                                "type": "prometheus",
                                "uid": "PBFA97CFB590B2093"
                            },
                            "editorMode": "code",
                            "expr": "container_memory_working_set_bytes{container=\"consul\", pod=~\"consul-client-.*\"}",
                            "hide": false,
                            "legendFormat": "__auto",
                            "range": true,
                            "refId": "A"
                        }
                    ],
                    "title": "Memory Usage (Consul clients)",
                    "type": "timeseries"
                },
                {
                    "datasource": {
                        "type": "prometheus",
                        "uid": "PBFA97CFB590B2093"
                    },
                    "description": "All consul servers have the same limit",
                    "fieldConfig": {
                        "defaults": {
                            "color": {
                                "mode": "continuous-BlYlRd"
                            },
                            "mappings": [],
                            "thresholds": {
                                "mode": "absolute",
                                "steps": [
                                    {
                                        "color": "green"
                                    },
                                    {
                                        "color": "red",
                                        "value": 80
                                    }
                                ]
                            },
                            "unit": "bytes"
                        },
                        "overrides": []
                    },
                    "gridPos": {
                        "h": 8,
                        "w": 9,
                        "x": 15,
                        "y": 29
                    },
                    "id": 25,
                    "options": {
                        "orientation": "auto",
                        "reduceOptions": {
                            "calcs": [
                                "lastNotNull"
                            ],
                            "fields": "",
                            "values": false
                        },
                        "showThresholdLabels": false,
                        "showThresholdMarkers": true
                    },
                    "pluginVersion": "9.5.5",
                    "targets": [
                        {
                            "datasource": {
                                "type": "prometheus",
                                "uid": "PBFA97CFB590B2093"
                            },
                            "editorMode": "code",
                            "exemplar": false,
                            "expr": "max(kube_pod_container_resource_limits{resource=\"memory\", pod=~\"consul-client-.*\"})",
                            "instant": true,
                            "legendFormat": "__auto",
                            "range": false,
                            "refId": "A"
                        }
                    ],
                    "title": "Memory Limit (Consul Clients)",
                    "type": "gauge"
                }
            ],
            "title": "Resource Utilization (Consul Clients) - N/A in agentless mode",
            "type": "row"
        },
        {
            "collapsed": true,
            "gridPos": {
                "h": 1,
                "w": 24,
                "x": 0,
                "y": 21
            },
            "id": 54,
            "panels": [
                {
                    "datasource": {
                        "type": "prometheus",
                        "uid": "PBFA97CFB590B2093"
                    },
                    "fieldConfig": {
                        "defaults": {
                            "color": {
                                "mode": "palette-classic"
                            },
                            "custom": {
                                "axisCenteredZero": false,
                                "axisColorMode": "text",
                                "axisLabel": "",
                                "axisPlacement": "auto",
                                "barAlignment": 0,
                                "drawStyle": "line",
                                "fillOpacity": 0,
                                "gradientMode": "none",
                                "hideFrom": {
                                    "legend": false,
                                    "tooltip": false,
                                    "viz": false
                                },
                                "lineInterpolation": "linear",
                                "lineWidth": 1,
                                "pointSize": 5,
                                "scaleDistribution": {
                                    "type": "linear"
                                },
                                "showPoints": "auto",
                                "spanNulls": false,
                                "stacking": {
                                    "group": "A",
                                    "mode": "none"
                                },
                                "thresholdsStyle": {
                                    "mode": "off"
                                }
                            },
                            "mappings": [],
                            "thresholds": {
                                "mode": "absolute",
                                "steps": [
                                    {
                                        "color": "green"
                                    },
                                    {
                                        "color": "red",
                                        "value": 80
                                    }
                                ]
                            }
                        },
                        "overrides": []
                    },
                    "gridPos": {
                        "h": 8,
                        "w": 15,
                        "x": 0,
                        "y": 25
                    },
                    "id": 55,
                    "options": {
                        "legend": {
                            "calcs": [],
                            "displayMode": "list",
                            "placement": "bottom",
                            "showLegend": true
                        },
                        "tooltip": {
                            "mode": "single",
                            "sort": "none"
                        }
                    },
                    "targets": [
                        {
                            "datasource": {
                                "type": "prometheus",
                                "uid": "PBFA97CFB590B2093"
                            },
                            "editorMode": "code",
                            "expr": "rate(container_cpu_usage_seconds_total{pod=~\".*-connect-injector-.*\",container=\"sidecar-injector\"}[5m])",
                            "hide": false,
                            "legendFormat": "__auto",
                            "range": true,
                            "refId": "A"
                        },
                        {
                            "datasource": {
                                "type": "prometheus",
                                "uid": "PBFA97CFB590B2093"
                            },
                            "editorMode": "builder",
                            "expr": "kube_pod_container_resource_limits{resource=\"cpu\", container=\"consul\"}",
                            "hide": true,
                            "legendFormat": "__auto",
                            "range": true,
                            "refId": "B"
                        }
                    ],
                    "title": "CPU Usage in Seconds (Connect Injector)",
                    "type": "timeseries"
                },
                {
                    "datasource": {
                        "type": "prometheus",
                        "uid": "PBFA97CFB590B2093"
                    },
                    "description": "All consul servers have the same limit",
                    "fieldConfig": {
                        "defaults": {
                            "color": {
                                "mode": "thresholds"
                            },
                            "mappings": [],
                            "thresholds": {
                                "mode": "absolute",
                                "steps": [
                                    {
                                        "color": "green"
                                    },
                                    {
                                        "color": "red",
                                        "value": 80
                                    }
                                ]
                            }
                        },
                        "overrides": []
                    },
                    "gridPos": {
                        "h": 8,
                        "w": 9,
                        "x": 15,
                        "y": 25
                    },
                    "id": 56,
                    "options": {
                        "orientation": "auto",
                        "reduceOptions": {
                            "calcs": [
                                "lastNotNull"
                            ],
                            "fields": "",
                            "values": false
                        },
                        "showThresholdLabels": false,
                        "showThresholdMarkers": true
                    },
                    "pluginVersion": "9.5.5",
                    "targets": [
                        {
                            "datasource": {
                                "type": "prometheus",
                                "uid": "PBFA97CFB590B2093"
                            },
                            "editorMode": "code",
                            "exemplar": false,
                            "expr": "max(kube_pod_container_resource_limits{resource=\"cpu\", container=\"sidecar-injector\"})",
                            "instant": true,
                            "legendFormat": "__auto",
                            "range": false,
                            "refId": "A"
                        }
                    ],
                    "title": "CPU Limit in Seconds (Connect Injector)",
                    "type": "gauge"
                },
                {
                    "datasource": {
                        "type": "prometheus",
                        "uid": "PBFA97CFB590B2093"
                    },
                    "description": "",
                    "fieldConfig": {
                        "defaults": {
                            "color": {
                                "mode": "palette-classic"
                            },
                            "custom": {
                                "axisCenteredZero": false,
                                "axisColorMode": "text",
                                "axisLabel": "",
                                "axisPlacement": "auto",
                                "barAlignment": 0,
                                "drawStyle": "line",
                                "fillOpacity": 0,
                                "gradientMode": "none",
                                "hideFrom": {
                                    "legend": false,
                                    "tooltip": false,
                                    "viz": false
                                },
                                "lineInterpolation": "linear",
                                "lineWidth": 1,
                                "pointSize": 5,
                                "scaleDistribution": {
                                    "type": "linear"
                                },
                                "showPoints": "auto",
                                "spanNulls": false,
                                "stacking": {
                                    "group": "A",
                                    "mode": "none"
                                },
                                "thresholdsStyle": {
                                    "mode": "off"
                                }
                            },
                            "mappings": [],
                            "thresholds": {
                                "mode": "absolute",
                                "steps": [
                                    {
                                        "color": "green"
                                    },
                                    {
                                        "color": "red",
                                        "value": 80
                                    }
                                ]
                            },
                            "unit": "bytes"
                        },
                        "overrides": []
                    },
                    "gridPos": {
                        "h": 8,
                        "w": 15,
                        "x": 0,
                        "y": 33
                    },
                    "id": 59,
                    "options": {
                        "legend": {
                            "calcs": [],
                            "displayMode": "list",
                            "placement": "bottom",
                            "showLegend": true
                        },
                        "tooltip": {
                            "mode": "single",
                            "sort": "none"
                        }
                    },
                    "targets": [
                        {
                            "datasource": {
                                "type": "prometheus",
                                "uid": "PBFA97CFB590B2093"
                            },
                            "editorMode": "code",
                            "expr": "container_memory_working_set_bytes{pod=~\".*-connect-injector-.*\",container=\"sidecar-injector\"}",
                            "hide": false,
                            "legendFormat": "__auto",
                            "range": true,
                            "refId": "A"
                        }
                    ],
                    "title": "Memory Usage (Connect Injector)",
                    "type": "timeseries"
                },
                {
                    "datasource": {
                        "type": "prometheus",
                        "uid": "PBFA97CFB590B2093"
                    },
                    "description": "All consul servers have the same limit",
                    "fieldConfig": {
                        "defaults": {
                            "color": {
                                "mode": "continuous-BlYlRd"
                            },
                            "mappings": [],
                            "thresholds": {
                                "mode": "absolute",
                                "steps": [
                                    {
                                        "color": "green"
                                    },
                                    {
                                        "color": "red",
                                        "value": 80
                                    }
                                ]
                            },
                            "unit": "bytes"
                        },
                        "overrides": []
                    },
                    "gridPos": {
                        "h": 8,
                        "w": 9,
                        "x": 15,
                        "y": 33
                    },
                    "id": 58,
                    "options": {
                        "orientation": "auto",
                        "reduceOptions": {
                            "calcs": [
                                "lastNotNull"
                            ],
                            "fields": "",
                            "values": false
                        },
                        "showThresholdLabels": false,
                        "showThresholdMarkers": true
                    },
                    "pluginVersion": "9.5.5",
                    "targets": [
                        {
                            "datasource": {
                                "type": "prometheus",
                                "uid": "PBFA97CFB590B2093"
                            },
                            "editorMode": "code",
                            "exemplar": false,
                            "expr": "max(kube_pod_container_resource_limits{resource=\"memory\", container=\"sidecar-injector\"})",
                            "instant": true,
                            "legendFormat": "__auto",
                            "range": false,
                            "refId": "A"
                        }
                    ],
                    "title": "Memory Limit (Connect Injector)",
                    "type": "gauge"
                }
            ],
            "title": "Resource Utilization (Connect Injector)",
            "type": "row"
        },
        {
            "collapsed": true,
            "gridPos": {
                "h": 1,
                "w": 24,
                "x": 0,
                "y": 22
            },
            "id": 30,
            "panels": [
                {
                    "datasource": {
                        "type": "prometheus",
                        "uid": "PBFA97CFB590B2093"
                    },
                    "fieldConfig": {
                        "defaults": {
                            "color": {
                                "mode": "palette-classic"
                            },
                            "custom": {
                                "axisCenteredZero": false,
                                "axisColorMode": "text",
                                "axisLabel": "",
                                "axisPlacement": "auto",
                                "barAlignment": 0,
                                "drawStyle": "line",
                                "fillOpacity": 0,
                                "gradientMode": "none",
                                "hideFrom": {
                                    "legend": false,
                                    "tooltip": false,
                                    "viz": false
                                },
                                "lineInterpolation": "linear",
                                "lineWidth": 1,
                                "pointSize": 5,
                                "scaleDistribution": {
                                    "type": "linear"
                                },
                                "showPoints": "auto",
                                "spanNulls": false,
                                "stacking": {
                                    "group": "A",
                                    "mode": "none"
                                },
                                "thresholdsStyle": {
                                    "mode": "off"
                                }
                            },
                            "mappings": [],
                            "thresholds": {
                                "mode": "absolute",
                                "steps": [
                                    {
                                        "color": "green"
                                    },
                                    {
                                        "color": "red",
                                        "value": 80
                                    }
                                ]
                            }
                        },
                        "overrides": []
                    },
                    "gridPos": {
                        "h": 8,
                        "w": 12,
                        "x": 0,
                        "y": 23
                    },
                    "id": 31,
                    "options": {
                        "legend": {
                            "calcs": [],
                            "displayMode": "list",
                            "placement": "bottom",
                            "showLegend": true
                        },
                        "tooltip": {
                            "mode": "single",
                            "sort": "none"
                        }
                    },
                    "targets": [
                        {
                            "datasource": {
                                "type": "prometheus",
                                "uid": "PBFA97CFB590B2093"
                            },
                            "editorMode": "code",
                            "expr": "sum(rate(consul_catalog_register_count{pod=~\"consul-server-.*\"}[5m]))",
                            "legendFormat": "__auto",
                            "range": true,
                            "refId": "A"
                        }
                    ],
                    "title": "Catalog Register Count per 5 minutes",
                    "type": "timeseries"
                },
                {
                    "datasource": {
                        "type": "prometheus",
                        "uid": "PBFA97CFB590B2093"
                    },
                    "fieldConfig": {
                        "defaults": {
                            "color": {
                                "mode": "palette-classic"
                            },
                            "custom": {
                                "axisCenteredZero": false,
                                "axisColorMode": "text",
                                "axisLabel": "",
                                "axisPlacement": "auto",
                                "barAlignment": 0,
                                "drawStyle": "line",
                                "fillOpacity": 0,
                                "gradientMode": "none",
                                "hideFrom": {
                                    "legend": false,
                                    "tooltip": false,
                                    "viz": false
                                },
                                "lineInterpolation": "linear",
                                "lineWidth": 1,
                                "pointSize": 5,
                                "scaleDistribution": {
                                    "type": "linear"
                                },
                                "showPoints": "auto",
                                "spanNulls": false,
                                "stacking": {
                                    "group": "A",
                                    "mode": "none"
                                },
                                "thresholdsStyle": {
                                    "mode": "off"
                                }
                            },
                            "mappings": [],
                            "thresholds": {
                                "mode": "absolute",
                                "steps": [
                                    {
                                        "color": "green"
                                    },
                                    {
                                        "color": "red",
                                        "value": 80
                                    }
                                ]
                            }
                        },
                        "overrides": []
                    },
                    "gridPos": {
                        "h": 8,
                        "w": 12,
                        "x": 12,
                        "y": 23
                    },
                    "id": 34,
                    "options": {
                        "legend": {
                            "calcs": [],
                            "displayMode": "list",
                            "placement": "bottom",
                            "showLegend": true
                        },
                        "tooltip": {
                            "mode": "single",
                            "sort": "none"
                        }
                    },
                    "targets": [
                        {
                            "datasource": {
                                "type": "prometheus",
                                "uid": "PBFA97CFB590B2093"
                            },
                            "editorMode": "code",
                            "expr": "rate(consul_catalog_register_sum{pod=~\"consul-server-.*\"}[5m])\n/\nrate(consul_catalog_register_count{pod=~\"consul-server-.*\"}[5m])",
                            "legendFormat": "__auto",
                            "range": true,
                            "refId": "A"
                        }
                    ],
                    "title": "Average latency of catalog register per 5 minutes (ms)",
                    "type": "timeseries"
                }
            ],
            "title": "Feature: Catalog",
            "type": "row"
        },
        {
            "collapsed": true,
            "gridPos": {
                "h": 1,
                "w": 24,
                "x": 0,
                "y": 23
            },
            "id": 49,
            "panels": [
                {
                    "datasource": {
                        "type": "prometheus",
                        "uid": "PBFA97CFB590B2093"
                    },
                    "fieldConfig": {
                        "defaults": {
                            "color": {
                                "mode": "palette-classic"
                            },
                            "custom": {
                                "axisCenteredZero": false,
                                "axisColorMode": "text",
                                "axisLabel": "",
                                "axisPlacement": "auto",
                                "barAlignment": 0,
                                "drawStyle": "line",
                                "fillOpacity": 0,
                                "gradientMode": "none",
                                "hideFrom": {
                                    "legend": false,
                                    "tooltip": false,
                                    "viz": false
                                },
                                "lineInterpolation": "linear",
                                "lineWidth": 1,
                                "pointSize": 5,
                                "scaleDistribution": {
                                    "type": "linear"
                                },
                                "showPoints": "auto",
                                "spanNulls": false,
                                "stacking": {
                                    "group": "A",
                                    "mode": "none"
                                },
                                "thresholdsStyle": {
                                    "mode": "off"
                                }
                            },
                            "mappings": [],
                            "thresholds": {
                                "mode": "absolute",
                                "steps": [
                                    {
                                        "color": "green"
                                    },
                                    {
                                        "color": "red",
                                        "value": 80
                                    }
                                ]
                            }
                        },
                        "overrides": []
                    },
                    "gridPos": {
                        "h": 8,
                        "w": 12,
                        "x": 0,
                        "y": 24
                    },
                    "id": 50,
                    "options": {
                        "legend": {
                            "calcs": [],
                            "displayMode": "list",
                            "placement": "bottom",
                            "showLegend": true
                        },
                        "tooltip": {
                            "mode": "single",
                            "sort": "none"
                        }
                    },
                    "targets": [
                        {
                            "datasource": {
                                "type": "prometheus",
                                "uid": "PBFA97CFB590B2093"
                            },
                            "editorMode": "code",
                            "expr": "sum(rate(consul_acl_ResolveToken_count{pod=~\"consul-server-.*\"}[5m]))",
                            "legendFormat": "__auto",
                            "range": true,
                            "refId": "A"
                        }
                    ],
                    "title": "ACL Token Resolve Count per 5 minutes",
                    "type": "timeseries"
                },
                {
                    "datasource": {
                        "type": "prometheus",
                        "uid": "PBFA97CFB590B2093"
                    },
                    "fieldConfig": {
                        "defaults": {
                            "color": {
                                "mode": "palette-classic"
                            },
                            "custom": {
                                "axisCenteredZero": false,
                                "axisColorMode": "text",
                                "axisLabel": "",
                                "axisPlacement": "auto",
                                "barAlignment": 0,
                                "drawStyle": "line",
                                "fillOpacity": 0,
                                "gradientMode": "none",
                                "hideFrom": {
                                    "legend": false,
                                    "tooltip": false,
                                    "viz": false
                                },
                                "lineInterpolation": "linear",
                                "lineWidth": 1,
                                "pointSize": 5,
                                "scaleDistribution": {
                                    "type": "linear"
                                },
                                "showPoints": "auto",
                                "spanNulls": false,
                                "stacking": {
                                    "group": "A",
                                    "mode": "none"
                                },
                                "thresholdsStyle": {
                                    "mode": "off"
                                }
                            },
                            "mappings": [],
                            "thresholds": {
                                "mode": "absolute",
                                "steps": [
                                    {
                                        "color": "green"
                                    },
                                    {
                                        "color": "red",
                                        "value": 80
                                    }
                                ]
                            }
                        },
                        "overrides": []
                    },
                    "gridPos": {
                        "h": 8,
                        "w": 12,
                        "x": 12,
                        "y": 24
                    },
                    "id": 51,
                    "options": {
                        "legend": {
                            "calcs": [],
                            "displayMode": "list",
                            "placement": "bottom",
                            "showLegend": true
                        },
                        "tooltip": {
                            "mode": "single",
                            "sort": "none"
                        }
                    },
                    "targets": [
                        {
                            "datasource": {
                                "type": "prometheus",
                                "uid": "PBFA97CFB590B2093"
                            },
                            "editorMode": "code",
                            "expr": "rate(consul_acl_ResolveToken_sum{pod=~\"consul-server-.*\"}[5m])/rate(consul_acl_ResolveToken_count{pod=~\"consul-server-.*\"}[5m])",
                            "legendFormat": "__auto",
                            "range": true,
                            "refId": "A"
                        }
                    ],
                    "title": "Average latency of resolving ACL token per 5 minutes (ms)",
                    "type": "timeseries"
                },
                {
                    "datasource": {
                        "type": "prometheus",
                        "uid": "PBFA97CFB590B2093"
                    },
                    "description": "",
                    "fieldConfig": {
                        "defaults": {
                            "color": {
                                "mode": "palette-classic"
                            },
                            "custom": {
                                "axisCenteredZero": false,
                                "axisColorMode": "text",
                                "axisLabel": "",
                                "axisPlacement": "auto",
                                "barAlignment": 0,
                                "drawStyle": "line",
                                "fillOpacity": 0,
                                "gradientMode": "none",
                                "hideFrom": {
                                    "legend": false,
                                    "tooltip": false,
                                    "viz": false
                                },
                                "lineInterpolation": "linear",
                                "lineWidth": 1,
                                "pointSize": 5,
                                "scaleDistribution": {
                                    "type": "linear"
                                },
                                "showPoints": "auto",
                                "spanNulls": false,
                                "stacking": {
                                    "group": "A",
                                    "mode": "none"
                                },
                                "thresholdsStyle": {
                                    "mode": "off"
                                }
                            },
                            "mappings": [],
                            "thresholds": {
                                "mode": "absolute",
                                "steps": [
                                    {
                                        "color": "green"
                                    },
                                    {
                                        "color": "red",
                                        "value": 80
                                    }
                                ]
                            }
                        },
                        "overrides": []
                    },
                    "gridPos": {
                        "h": 8,
                        "w": 12,
                        "x": 0,
                        "y": 32
                    },
                    "id": 52,
                    "options": {
                        "legend": {
                            "calcs": [],
                            "displayMode": "list",
                            "placement": "bottom",
                            "showLegend": true
                        },
                        "tooltip": {
                            "mode": "single",
                            "sort": "none"
                        }
                    },
                    "targets": [
                        {
                            "datasource": {
                                "type": "prometheus",
                                "uid": "PBFA97CFB590B2093"
                            },
                            "editorMode": "code",
                            "expr": "rate(consul_acl_token_cache_hit{pod=~\"consul-server-.*\"}[5m])",
                            "legendFormat": "__auto",
                            "range": true,
                            "refId": "A"
                        }
                    ],
                    "title": "Rate of ACL token cache hit per 5 minutes",
                    "type": "timeseries"
                },
                {
                    "datasource": {
                        "type": "prometheus",
                        "uid": "PBFA97CFB590B2093"
                    },
                    "description": "",
                    "fieldConfig": {
                        "defaults": {
                            "color": {
                                "mode": "palette-classic"
                            },
                            "custom": {
                                "axisCenteredZero": false,
                                "axisColorMode": "text",
                                "axisLabel": "",
                                "axisPlacement": "auto",
                                "barAlignment": 0,
                                "drawStyle": "line",
                                "fillOpacity": 0,
                                "gradientMode": "none",
                                "hideFrom": {
                                    "legend": false,
                                    "tooltip": false,
                                    "viz": false
                                },
                                "lineInterpolation": "linear",
                                "lineWidth": 1,
                                "pointSize": 5,
                                "scaleDistribution": {
                                    "type": "linear"
                                },
                                "showPoints": "auto",
                                "spanNulls": false,
                                "stacking": {
                                    "group": "A",
                                    "mode": "none"
                                },
                                "thresholdsStyle": {
                                    "mode": "off"
                                }
                            },
                            "mappings": [],
                            "thresholds": {
                                "mode": "absolute",
                                "steps": [
                                    {
                                        "color": "green"
                                    },
                                    {
                                        "color": "red",
                                        "value": 80
                                    }
                                ]
                            }
                        },
                        "overrides": []
                    },
                    "gridPos": {
                        "h": 8,
                        "w": 12,
                        "x": 12,
                        "y": 32
                    },
                    "id": 53,
                    "options": {
                        "legend": {
                            "calcs": [],
                            "displayMode": "list",
                            "placement": "bottom",
                            "showLegend": true
                        },
                        "tooltip": {
                            "mode": "single",
                            "sort": "none"
                        }
                    },
                    "targets": [
                        {
                            "datasource": {
                                "type": "prometheus",
                                "uid": "PBFA97CFB590B2093"
                            },
                            "editorMode": "code",
                            "expr": "rate(consul_acl_token_cache_miss{pod=~\"consul-server-.*\"}[5m])",
                            "legendFormat": "__auto",
                            "range": true,
                            "refId": "A"
                        }
                    ],
                    "title": "Rate of ACL token cache miss per 5 minutes",
                    "type": "timeseries"
                }
            ],
            "title": "ACL",
            "type": "row"
        },
        {
            "collapsed": true,
            "gridPos": {
                "h": 1,
                "w": 24,
                "x": 0,
                "y": 24
            },
            "id": 33,
            "panels": [
                {
                    "datasource": {
                        "type": "prometheus",
                        "uid": "PBFA97CFB590B2093"
                    },
                    "description": "Increments whenever a Consul agent in client mode makes an RPC request to a Consul server",
                    "fieldConfig": {
                        "defaults": {
                            "color": {
                                "mode": "palette-classic"
                            },
                            "custom": {
                                "axisCenteredZero": false,
                                "axisColorMode": "text",
                                "axisLabel": "",
                                "axisPlacement": "auto",
                                "barAlignment": 0,
                                "drawStyle": "line",
                                "fillOpacity": 0,
                                "gradientMode": "none",
                                "hideFrom": {
                                    "legend": false,
                                    "tooltip": false,
                                    "viz": false
                                },
                                "lineInterpolation": "linear",
                                "lineWidth": 1,
                                "pointSize": 5,
                                "scaleDistribution": {
                                    "type": "linear"
                                },
                                "showPoints": "auto",
                                "spanNulls": false,
                                "stacking": {
                                    "group": "A",
                                    "mode": "none"
                                },
                                "thresholdsStyle": {
                                    "mode": "off"
                                }
                            },
                            "mappings": [],
                            "thresholds": {
                                "mode": "absolute",
                                "steps": [
                                    {
                                        "color": "green"
                                    },
                                    {
                                        "color": "red",
                                        "value": 80
                                    }
                                ]
                            }
                        },
                        "overrides": []
                    },
                    "gridPos": {
                        "h": 8,
                        "w": 12,
                        "x": 0,
                        "y": 22
                    },
                    "id": 37,
                    "options": {
                        "legend": {
                            "calcs": [],
                            "displayMode": "list",
                            "placement": "bottom",
                            "showLegend": true
                        },
                        "tooltip": {
                            "mode": "single",
                            "sort": "none"
                        }
                    },
                    "targets": [
                        {
                            "datasource": {
                                "type": "prometheus",
                                "uid": "PBFA97CFB590B2093"
                            },
                            "editorMode": "code",
                            "expr": "rate(consul_client_rpc{namespace=\"consul\"}[5m])",
                            "legendFormat": "__auto",
                            "range": true,
                            "refId": "A"
                        }
                    ],
                    "title": "Rate of RPC requests per 5 minutes - client side ",
                    "type": "timeseries"
                },
                {
                    "datasource": {
                        "type": "prometheus",
                        "uid": "PBFA97CFB590B2093"
                    },
                    "description": "Increments when a server accepts an RPC connection.",
                    "fieldConfig": {
                        "defaults": {
                            "color": {
                                "mode": "palette-classic"
                            },
                            "custom": {
                                "axisCenteredZero": false,
                                "axisColorMode": "text",
                                "axisLabel": "",
                                "axisPlacement": "auto",
                                "barAlignment": 0,
                                "drawStyle": "line",
                                "fillOpacity": 0,
                                "gradientMode": "none",
                                "hideFrom": {
                                    "legend": false,
                                    "tooltip": false,
                                    "viz": false
                                },
                                "lineInterpolation": "linear",
                                "lineWidth": 1,
                                "pointSize": 5,
                                "scaleDistribution": {
                                    "type": "linear"
                                },
                                "showPoints": "auto",
                                "spanNulls": false,
                                "stacking": {
                                    "group": "A",
                                    "mode": "none"
                                },
                                "thresholdsStyle": {
                                    "mode": "off"
                                }
                            },
                            "mappings": [],
                            "thresholds": {
                                "mode": "absolute",
                                "steps": [
                                    {
                                        "color": "green"
                                    },
                                    {
                                        "color": "red",
                                        "value": 80
                                    }
                                ]
                            }
                        },
                        "overrides": []
                    },
                    "gridPos": {
                        "h": 8,
                        "w": 12,
                        "x": 12,
                        "y": 22
                    },
                    "id": 36,
                    "options": {
                        "legend": {
                            "calcs": [],
                            "displayMode": "list",
                            "placement": "bottom",
                            "showLegend": true
                        },
                        "tooltip": {
                            "mode": "single",
                            "sort": "none"
                        }
                    },
                    "targets": [
                        {
                            "datasource": {
                                "type": "prometheus",
                                "uid": "PBFA97CFB590B2093"
                            },
                            "editorMode": "code",
                            "expr": "rate(consul_rpc_accept_conn{namespace=\"consul\",pod=~\"consul-server-.*\"}[5m])",
                            "legendFormat": "__auto",
                            "range": true,
                            "refId": "A"
                        }
                    ],
                    "title": "RPC Accept Connection Count Rate per 5 minutes - server side",
                    "type": "timeseries"
                },
                {
                    "datasource": {
                        "type": "prometheus",
                        "uid": "PBFA97CFB590B2093"
                    },
                    "description": "Increments whenever a Consul agent in client mode makes an RPC request to a Consul server and fails.",
                    "fieldConfig": {
                        "defaults": {
                            "color": {
                                "mode": "palette-classic"
                            },
                            "custom": {
                                "axisCenteredZero": false,
                                "axisColorMode": "text",
                                "axisLabel": "",
                                "axisPlacement": "auto",
                                "barAlignment": 0,
                                "drawStyle": "line",
                                "fillOpacity": 0,
                                "gradientMode": "none",
                                "hideFrom": {
                                    "legend": false,
                                    "tooltip": false,
                                    "viz": false
                                },
                                "lineInterpolation": "linear",
                                "lineWidth": 1,
                                "pointSize": 5,
                                "scaleDistribution": {
                                    "type": "linear"
                                },
                                "showPoints": "auto",
                                "spanNulls": false,
                                "stacking": {
                                    "group": "A",
                                    "mode": "none"
                                },
                                "thresholdsStyle": {
                                    "mode": "off"
                                }
                            },
                            "mappings": [],
                            "thresholds": {
                                "mode": "absolute",
                                "steps": [
                                    {
                                        "color": "green"
                                    },
                                    {
                                        "color": "red",
                                        "value": 80
                                    }
                                ]
                            }
                        },
                        "overrides": []
                    },
                    "gridPos": {
                        "h": 8,
                        "w": 12,
                        "x": 0,
                        "y": 30
                    },
                    "id": 39,
                    "options": {
                        "legend": {
                            "calcs": [],
                            "displayMode": "list",
                            "placement": "bottom",
                            "showLegend": true
                        },
                        "tooltip": {
                            "mode": "single",
                            "sort": "none"
                        }
                    },
                    "targets": [
                        {
                            "datasource": {
                                "type": "prometheus",
                                "uid": "PBFA97CFB590B2093"
                            },
                            "editorMode": "code",
                            "expr": "rate(consul_client_rpc_failed{namespace=\"consul\"}[5m])",
                            "legendFormat": "__auto",
                            "range": true,
                            "refId": "A"
                        }
                    ],
                    "title": "Rate of Failed RPC requests per 5 minutes - client side ",
                    "type": "timeseries"
                },
                {
                    "datasource": {
                        "type": "prometheus",
                        "uid": "PBFA97CFB590B2093"
                    },
                    "description": "Increments when a server receives a Consul-related RPC request.",
                    "fieldConfig": {
                        "defaults": {
                            "color": {
                                "mode": "palette-classic"
                            },
                            "custom": {
                                "axisCenteredZero": false,
                                "axisColorMode": "text",
                                "axisLabel": "",
                                "axisPlacement": "auto",
                                "barAlignment": 0,
                                "drawStyle": "line",
                                "fillOpacity": 0,
                                "gradientMode": "none",
                                "hideFrom": {
                                    "legend": false,
                                    "tooltip": false,
                                    "viz": false
                                },
                                "lineInterpolation": "linear",
                                "lineWidth": 1,
                                "pointSize": 5,
                                "scaleDistribution": {
                                    "type": "linear"
                                },
                                "showPoints": "auto",
                                "spanNulls": false,
                                "stacking": {
                                    "group": "A",
                                    "mode": "none"
                                },
                                "thresholdsStyle": {
                                    "mode": "off"
                                }
                            },
                            "mappings": [],
                            "thresholds": {
                                "mode": "absolute",
                                "steps": [
                                    {
                                        "color": "green"
                                    },
                                    {
                                        "color": "red",
                                        "value": 80
                                    }
                                ]
                            }
                        },
                        "overrides": []
                    },
                    "gridPos": {
                        "h": 8,
                        "w": 12,
                        "x": 12,
                        "y": 30
                    },
                    "id": 32,
                    "options": {
                        "legend": {
                            "calcs": [],
                            "displayMode": "list",
                            "placement": "bottom",
                            "showLegend": true
                        },
                        "tooltip": {
                            "mode": "single",
                            "sort": "none"
                        }
                    },
                    "targets": [
                        {
                            "datasource": {
                                "type": "prometheus",
                                "uid": "PBFA97CFB590B2093"
                            },
                            "editorMode": "code",
                            "expr": "rate(consul_rpc_request{namespace=\"consul\",pod=~\"consul-server-.*\"}[5m])",
                            "legendFormat": "__auto",
                            "range": true,
                            "refId": "A"
                        }
                    ],
                    "title": "Rate of RPC requests per 5 minutes - server side",
                    "type": "timeseries"
                },
                {
                    "datasource": {
                        "type": "prometheus",
                        "uid": "PBFA97CFB590B2093"
                    },
                    "description": "Increments whenever a Consul agent in client mode makes an RPC request to a Consul server gets rate limited by that agent's limits configuration.",
                    "fieldConfig": {
                        "defaults": {
                            "color": {
                                "mode": "palette-classic"
                            },
                            "custom": {
                                "axisCenteredZero": false,
                                "axisColorMode": "text",
                                "axisLabel": "",
                                "axisPlacement": "auto",
                                "barAlignment": 0,
                                "drawStyle": "line",
                                "fillOpacity": 0,
                                "gradientMode": "none",
                                "hideFrom": {
                                    "legend": false,
                                    "tooltip": false,
                                    "viz": false
                                },
                                "lineInterpolation": "linear",
                                "lineWidth": 1,
                                "pointSize": 5,
                                "scaleDistribution": {
                                    "type": "linear"
                                },
                                "showPoints": "auto",
                                "spanNulls": false,
                                "stacking": {
                                    "group": "A",
                                    "mode": "none"
                                },
                                "thresholdsStyle": {
                                    "mode": "off"
                                }
                            },
                            "mappings": [],
                            "thresholds": {
                                "mode": "absolute",
                                "steps": [
                                    {
                                        "color": "green"
                                    },
                                    {
                                        "color": "red",
                                        "value": 80
                                    }
                                ]
                            }
                        },
                        "overrides": []
                    },
                    "gridPos": {
                        "h": 8,
                        "w": 12,
                        "x": 0,
                        "y": 38
                    },
                    "id": 38,
                    "options": {
                        "legend": {
                            "calcs": [],
                            "displayMode": "list",
                            "placement": "bottom",
                            "showLegend": true
                        },
                        "tooltip": {
                            "mode": "single",
                            "sort": "none"
                        }
                    },
                    "targets": [
                        {
                            "datasource": {
                                "type": "prometheus",
                                "uid": "PBFA97CFB590B2093"
                            },
                            "editorMode": "code",
                            "expr": "rate(consul_client_rpc_exceeded{namespace=\"consul\"}[5m])",
                            "legendFormat": "__auto",
                            "range": true,
                            "refId": "A"
                        }
                    ],
                    "title": "Rate of Exceeded RPC requests per 5 minutes - client side ",
                    "type": "timeseries"
                },
                {
                    "datasource": {
                        "type": "prometheus",
                        "uid": "PBFA97CFB590B2093"
                    },
                    "description": "Increments when a server returns an error from an RPC request.",
                    "fieldConfig": {
                        "defaults": {
                            "color": {
                                "mode": "palette-classic"
                            },
                            "custom": {
                                "axisCenteredZero": false,
                                "axisColorMode": "text",
                                "axisLabel": "",
                                "axisPlacement": "auto",
                                "barAlignment": 0,
                                "drawStyle": "line",
                                "fillOpacity": 0,
                                "gradientMode": "none",
                                "hideFrom": {
                                    "legend": false,
                                    "tooltip": false,
                                    "viz": false
                                },
                                "lineInterpolation": "linear",
                                "lineWidth": 1,
                                "pointSize": 5,
                                "scaleDistribution": {
                                    "type": "linear"
                                },
                                "showPoints": "auto",
                                "spanNulls": false,
                                "stacking": {
                                    "group": "A",
                                    "mode": "none"
                                },
                                "thresholdsStyle": {
                                    "mode": "off"
                                }
                            },
                            "mappings": [],
                            "thresholds": {
                                "mode": "absolute",
                                "steps": [
                                    {
                                        "color": "green"
                                    },
                                    {
                                        "color": "red",
                                        "value": 80
                                    }
                                ]
                            }
                        },
                        "overrides": []
                    },
                    "gridPos": {
                        "h": 8,
                        "w": 12,
                        "x": 12,
                        "y": 38
                    },
                    "id": 35,
                    "options": {
                        "legend": {
                            "calcs": [],
                            "displayMode": "list",
                            "placement": "bottom",
                            "showLegend": true
                        },
                        "tooltip": {
                            "mode": "single",
                            "sort": "none"
                        }
                    },
                    "targets": [
                        {
                            "datasource": {
                                "type": "prometheus",
                                "uid": "PBFA97CFB590B2093"
                            },
                            "editorMode": "code",
                            "expr": "rate(consul_rpc_request_error{namespace=\"consul\",pod=~\"consul-server-.*\"}[5m])",
                            "legendFormat": "__auto",
                            "range": true,
                            "refId": "A"
                        }
                    ],
                    "title": "Error rate of RPC requests per 5 minutes - server side",
                    "type": "timeseries"
                }
            ],
            "title": "RPC",
            "type": "row"
        }
    ],
    "refresh": "30s",
    "revision": 1,
    "schemaVersion": 38,
    "style": "dark",
    "tags": [],
    "templating": {
        "list": []
    },
    "time": {
        "from": "now-30m",
        "to": "now"
    },
    "timepicker": {},
    "timezone": "",
    "title": "Consul K8s monitoring (control plane)",
    "version": 8,
    "weekStart": ""
}