{ "annotations": { "list": [ { "builtIn": 1, "datasource": { "type": "grafana", "uid": "-- Grafana --" }, "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", "name": "Annotations & Alerts", "target": { "limit": 100, "matchAny": false, "tags": [], "type": "dashboard" }, "type": "dashboard" } ] }, "description": "Application level metrics for the Keystone API server.", "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 0, "id": 2, "links": [], "panels": [ { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, "id": 17, "panels": [], "title": "Health Checks", "type": "row" }, { "datasource": { "default": false, "type": "prometheus", "uid": "${datasource}" }, "description": "Connection status between the application and the file storage backend.", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [ { "options": { "200": { "color": "green", "index": 0, "text": "OK" } }, "type": "value" }, { "options": { "from": 300, "result": { "color": "red", "index": 1, "text": "Failing" }, "to": 500 }, "type": "range" } ], "thresholds": { "mode": "absolute", "steps": [ { "color": "red" } ] } }, "overrides": [] }, "gridPos": { "h": 3, "w": 4, "x": 0, "y": 1 }, "id": 22, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showPercentChange": false, "textMode": "auto", "wideLayout": true }, "pluginVersion": "12.0.2", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "disableTextWrap": false, "editorMode": "code", "exemplar": false, "expr": "keystone_health_check_status{\n check=\"Storage\",\n namespace=\"$namespace\",\n service=\"keystone-api-health\"\n}", "fullMetaSearch": false, "includeNullMetadata": true, "instant": true, "legendFormat": "__auto", "range": false, "refId": "A", "useBackend": false } ], "title": "Storage Connection", "type": "stat" }, { "datasource": { "default": false, "type": "prometheus", "uid": "${datasource}" }, "description": "Connection status between the application and the database.", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [ { "options": { "200": { "color": "green", "index": 0, "text": "OK" } }, "type": "value" }, { "options": { "from": 300, "result": { "color": "red", "index": 1, "text": "Failing" }, "to": 600 }, "type": "range" } ], "thresholds": { "mode": "absolute", "steps": [ { "color": "red" } ] } }, "overrides": [] }, "gridPos": { "h": 3, "w": 4, "x": 4, "y": 1 }, "id": 18, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showPercentChange": false, "textMode": "auto", "wideLayout": true }, "pluginVersion": "12.0.2", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "disableTextWrap": false, "editorMode": "code", "exemplar": false, "expr": "keystone_health_check_status{\n check=\"Database\",\n namespace=~\"$namespace\",\n service=\"keystone-api-health\"\n}", "fullMetaSearch": false, "includeNullMetadata": true, "instant": true, "legendFormat": "__auto", "range": false, "refId": "A", "useBackend": false } ], "title": "Database Connection", "type": "stat" }, { "datasource": { "default": false, "type": "prometheus", "uid": "${datasource}" }, "description": "Connection status between the application and the Celery task manager.", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [ { "options": { "200": { "color": "green", "index": 0, "text": "OK" } }, "type": "value" }, { "options": { "from": 300, "result": { "color": "red", "index": 1, "text": "Failing" }, "to": 600 }, "type": "range" } ], "thresholds": { "mode": "absolute", "steps": [ { "color": "red" } ] } }, "overrides": [] }, "gridPos": { "h": 3, "w": 4, "x": 8, "y": 1 }, "id": 23, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showPercentChange": false, "textMode": "auto", "wideLayout": true }, "pluginVersion": "12.0.2", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "disableTextWrap": false, "editorMode": "code", "exemplar": false, "expr": "keystone_health_check_status{\n check=\"Celery\",\n namespace=\"$namespace\",\n service=\"keystone-api-health\"\n}", "fullMetaSearch": false, "includeNullMetadata": true, "instant": true, "legendFormat": "__auto", "range": false, "refId": "A", "useBackend": false } ], "title": "Celery Connection", "type": "stat" }, { "datasource": { "default": false, "type": "prometheus", "uid": "${datasource}" }, "description": "Connection status between the application and the Redis cache.", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [ { "options": { "200": { "color": "green", "index": 0, "text": "OK" } }, "type": "value" }, { "options": { "from": 300, "result": { "color": "red", "index": 1, "text": "Failing" }, "to": 600 }, "type": "range" } ], "thresholds": { "mode": "absolute", "steps": [ { "color": "red" } ] } }, "overrides": [] }, "gridPos": { "h": 3, "w": 4, "x": 12, "y": 1 }, "id": 20, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showPercentChange": false, "textMode": "auto", "wideLayout": true }, "pluginVersion": "12.0.2", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "disableTextWrap": false, "editorMode": "code", "exemplar": false, "expr": "keystone_health_check_status{\n check=\"Redis\",\n namespace=\"$namespace\",\n service=\"keystone-api-health\"\n}", "fullMetaSearch": false, "includeNullMetadata": true, "instant": true, "legendFormat": "__auto", "range": false, "refId": "A", "useBackend": false } ], "title": "Redis Connection", "type": "stat" }, { "datasource": { "default": false, "type": "prometheus", "uid": "${datasource}" }, "description": "Connection status between the application and the SMTP server.", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [ { "options": { "200": { "color": "green", "index": 0, "text": "OK" } }, "type": "value" }, { "options": { "from": 300, "result": { "color": "red", "index": 1, "text": "Failing" }, "to": 500 }, "type": "range" } ], "thresholds": { "mode": "absolute", "steps": [ { "color": "red" } ] } }, "overrides": [] }, "gridPos": { "h": 3, "w": 4, "x": 16, "y": 1 }, "id": 36, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showPercentChange": false, "textMode": "auto", "wideLayout": true }, "pluginVersion": "12.0.2", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "disableTextWrap": false, "editorMode": "code", "exemplar": false, "expr": "keystone_health_check_status{\n check=\"SMTP\",\n namespace=\"$namespace\",\n service=\"keystone-api-health\"\n}", "fullMetaSearch": false, "includeNullMetadata": true, "instant": true, "legendFormat": "__auto", "range": false, "refId": "A", "useBackend": false } ], "title": "SMTP Connection", "type": "stat" }, { "datasource": { "default": false, "type": "prometheus", "uid": "${datasource}" }, "description": "Connection status between the application and the LDAP server.", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [ { "options": { "200": { "color": "green", "index": 0, "text": "OK" } }, "type": "value" }, { "options": { "from": 300, "result": { "color": "red", "index": 1, "text": "Failing" }, "to": 500 }, "type": "range" } ], "thresholds": { "mode": "absolute", "steps": [ { "color": "red" } ] } }, "overrides": [] }, "gridPos": { "h": 3, "w": 4, "x": 20, "y": 1 }, "id": 38, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showPercentChange": false, "textMode": "auto", "wideLayout": true }, "pluginVersion": "12.0.2", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "disableTextWrap": false, "editorMode": "code", "exemplar": false, "expr": "keystone_health_check_status{\n check=\"LDAP\",\n namespace=\"$namespace\",\n service=\"keystone-api-health\"\n}", "fullMetaSearch": false, "includeNullMetadata": true, "instant": true, "legendFormat": "__auto", "range": false, "refId": "A", "useBackend": false } ], "title": "LDAP Connection", "type": "stat" }, { "datasource": { "name": "${datasource}", "type": "prometheus", "uid": "${datasource}" }, "description": "Connection status for individual services over time.", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "custom": { "axisPlacement": "auto", "fillOpacity": 70, "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineWidth": 0, "spanNulls": false }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "red" }, { "color": "green", "value": 200 }, { "color": "red", "value": 300 } ] } }, "overrides": [] }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 4 }, "id": 35, "options": { "alignValue": "left", "legend": { "displayMode": "list", "placement": "bottom", "showLegend": false }, "mergeValues": true, "rowHeight": 0.9, "showValue": "auto", "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "12.0.2", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "keystone_health_check_status{\n check=\"Storage\",\n namespace=~\"$namespace\",\n service=\"keystone-api-health\"\n}", "instant": false, "legendFormat": "Database Connection", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "keystone_health_check_status{\n check=\"Database\",\n namespace=\"$namespace\",\n service=\"keystone-api-health\"\n}", "hide": false, "instant": false, "legendFormat": "Storage Connection", "range": true, "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "keystone_health_check_status{\n check=\"Celery\",\n namespace=\"$namespace\",\n service=\"keystone-api-health\"\n}", "hide": false, "instant": false, "legendFormat": "Celery Connection", "range": true, "refId": "C" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "keystone_health_check_status{\n check=\"Redis\",\n namespace=\"$namespace\",\n service=\"keystone-api-health\"\n}", "hide": false, "instant": false, "legendFormat": "Redis Connection", "range": true, "refId": "D" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "keystone_health_check_status{\n check=\"SMTP\",\n namespace=\"$namespace\",\n service=\"keystone-api-health\"\n}", "hide": false, "instant": false, "legendFormat": "SMTP Connection", "range": true, "refId": "E" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "keystone_health_check_status{\n check=\"LDAP\",\n namespace=\"$namespace\",\n service=\"keystone-api-health\"\n}", "hide": false, "instant": false, "legendFormat": "SMTP Connection", "range": true, "refId": "F" } ], "title": "Connection Status", "type": "state-timeline" }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 11 }, "id": 14, "panels": [], "title": "API Requests", "type": "row" }, { "datasource": { "default": false, "type": "prometheus", "uid": "${datasource}" }, "description": "Total rate of incoming requests reaching the application.", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "text" } ] }, "unit": "req/s" }, "overrides": [] }, "gridPos": { "h": 4, "w": 6, "x": 0, "y": 12 }, "id": 2, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showPercentChange": false, "text": {}, "textMode": "auto", "wideLayout": true }, "pluginVersion": "12.0.2", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "round(\n sum(\n rate(\n django_http_requests_total_by_view_transport_method_total{\n namespace=\"$namespace\",\n service=\"keystone-api-metrics\",\n method=~\"$method\",\n view=~\"$view\"\n }[$__rate_interval]\n )\n ), 0.001\n)\n", "instant": false, "legendFormat": "__auto", "range": true, "refId": "A" } ], "title": "Incoming Requests", "type": "stat" }, { "datasource": { "default": false, "type": "prometheus", "uid": "${datasource}" }, "description": "Ratio of successful HTTP response codes issued by the application.", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "decimals": 1, "mappings": [ { "options": { "null": { "color": "red", "index": 0, "text": "N/A" } }, "type": "value" } ], "max": 1, "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "red" }, { "color": "#EAB839", "value": 0.9 }, { "color": "green", "value": 0.95 } ] }, "unit": "percentunit" }, "overrides": [] }, "gridPos": { "h": 4, "w": 6, "x": 6, "y": 12 }, "id": 3, "options": { "minVizHeight": 75, "minVizWidth": 75, "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showThresholdLabels": false, "showThresholdMarkers": true, "sizing": "auto" }, "pluginVersion": "12.0.2", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "sum(\n django_http_responses_total_by_status_view_method_total{\n namespace=\"$namespace\",\n service=\"keystone-api-metrics\",\n method=~\"$method\",\n view=~\"$view\",\n status!~\"[4-5].+\"\n }\n) /\nsum(\n django_http_responses_total_by_status_view_method_total{\n namespace=\"$namespace\",\n service=\"keystone-api-metrics\",\n method=~\"$method\",\n view=~\"$view\",\n } \n)\n", "instant": false, "legendFormat": "__auto", "range": true, "refId": "A" } ], "title": "Success Rate (non 4xx/5xx)", "type": "gauge" }, { "datasource": { "default": false, "type": "prometheus", "uid": "${datasource}" }, "description": "Percent of application responses with a 2XX status code.", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [ { "options": { "match": "null", "result": { "index": 0, "text": "0%" } }, "type": "special" } ], "thresholds": { "mode": "absolute", "steps": [ { "color": "text" } ] }, "unit": "percentunit" }, "overrides": [] }, "gridPos": { "h": 2, "w": 3, "x": 12, "y": 12 }, "id": 4, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showPercentChange": false, "textMode": "auto", "wideLayout": true }, "pluginVersion": "12.0.2", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "sum(\n django_http_responses_total_by_status_view_method_total{\n namespace=\"$namespace\",\n service=\"keystone-api-metrics\",\n method=~\"$method\",\n view=~\"$view\",\n status=~\"2.+\"\n }\n) /\nsum(\n django_http_responses_total_by_status_view_method_total{\n namespace=\"$namespace\",\n service=\"keystone-api-metrics\",\n method=~\"$method\",\n view=~\"$view\",\n } \n)\n", "instant": false, "legendFormat": "__auto", "range": true, "refId": "A" } ], "title": "2XX Ratio", "type": "stat" }, { "datasource": { "default": false, "type": "prometheus", "uid": "${datasource}" }, "description": "Percent of application responses with a 4XX status code.", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [ { "options": { "match": "null", "result": { "index": 0, "text": "0%" } }, "type": "special" } ], "thresholds": { "mode": "absolute", "steps": [ { "color": "text" } ] }, "unit": "percentunit" }, "overrides": [] }, "gridPos": { "h": 2, "w": 3, "x": 15, "y": 12 }, "id": 29, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showPercentChange": false, "textMode": "auto", "wideLayout": true }, "pluginVersion": "12.0.2", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "sum(\n django_http_responses_total_by_status_view_method_total{\n namespace=\"$namespace\",\n service=\"keystone-api-metrics\",\n method=~\"$method\",\n view=~\"$view\",\n status=~\"4.+\"\n }\n) /\nsum(\n django_http_responses_total_by_status_view_method_total{\n namespace=\"$namespace\",\n service=\"keystone-api-metrics\",\n method=~\"$method\",\n view=~\"$view\",\n } \n)\n", "instant": false, "legendFormat": "__auto", "range": true, "refId": "A" } ], "title": "4XX Ratio", "type": "stat" }, { "datasource": { "default": false, "type": "prometheus", "uid": "${datasource}" }, "description": "Size of the request body in bytes (95th percentile)", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "text" } ] }, "unit": "s" }, "overrides": [] }, "gridPos": { "h": 4, "w": 6, "x": 18, "y": 12 }, "id": 8, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showPercentChange": false, "textMode": "auto", "wideLayout": true }, "pluginVersion": "12.0.2", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "histogram_quantile(0.95, \n sum(\n rate(\n django_http_requests_latency_seconds_by_view_method_bucket{\n namespace=\"$namespace\",\n service=\"keystone-api-metrics\",\n method=~\"$method\",\n view=~\"$view\"\n }[$__rate_interval]\n )\n ) by (le)\n)", "instant": false, "legendFormat": "__auto", "range": true, "refId": "A" } ], "title": "Request Latentcy (P95)", "type": "stat" }, { "datasource": { "default": false, "type": "prometheus", "uid": "${datasource}" }, "description": "Percent of application responses with a 3XX status code.", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [ { "options": { "match": "null", "result": { "index": 0, "text": "0%" } }, "type": "special" } ], "thresholds": { "mode": "absolute", "steps": [ { "color": "text" } ] }, "unit": "percentunit" }, "overrides": [] }, "gridPos": { "h": 2, "w": 3, "x": 12, "y": 14 }, "id": 30, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showPercentChange": false, "textMode": "auto", "wideLayout": true }, "pluginVersion": "12.0.2", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "sum(\n django_http_responses_total_by_status_view_method_total{\n namespace=\"$namespace\",\n service=\"keystone-api-metrics\",\n method=~\"$method\",\n view=~\"$view\",\n status=~\"3.+\"\n }\n) /\nsum(\n django_http_responses_total_by_status_view_method_total{\n namespace=\"$namespace\",\n service=\"keystone-api-metrics\",\n method=~\"$method\",\n view=~\"$view\",\n } \n)\n", "instant": false, "legendFormat": "__auto", "range": true, "refId": "A" } ], "title": "3XX Ratio", "type": "stat" }, { "datasource": { "default": false, "type": "prometheus", "uid": "${datasource}" }, "description": "Percent of application responses with a 5XX status code.", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [ { "options": { "match": "null", "result": { "index": 0, "text": "0%" } }, "type": "special" } ], "thresholds": { "mode": "absolute", "steps": [ { "color": "text" } ] }, "unit": "percentunit" }, "overrides": [] }, "gridPos": { "h": 2, "w": 3, "x": 15, "y": 14 }, "id": 28, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showPercentChange": false, "textMode": "auto", "wideLayout": true }, "pluginVersion": "12.0.2", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "sum(\n django_http_responses_total_by_status_view_method_total{\n namespace=\"$namespace\",\n service=\"keystone-api-metrics\",\n method=~\"$method\",\n view=~\"$view\",\n status=~\"5.+\"\n }\n) /\nsum(\n django_http_responses_total_by_status_view_method_total{\n namespace=\"$namespace\",\n service=\"keystone-api-metrics\",\n method=~\"$method\",\n view=~\"$view\",\n } \n)\n", "instant": false, "legendFormat": "__auto", "range": true, "refId": "A" } ], "title": "5XX Ratio", "type": "stat" }, { "datasource": { "default": false, "type": "prometheus", "uid": "${datasource}" }, "description": "Total rate of incoming requests reaching the application.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "text" } ] }, "unit": "req/s" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 16 }, "id": 9, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "12.0.2", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "sum(\n rate(\n django_http_requests_total_by_view_transport_method_total{\n namespace=\"$namespace\",\n service=\"keystone-api-metrics\",\n method=~\"$method\",\n view=~\"$view\"\n }[$__rate_interval]\n )\n)\n", "instant": false, "legendFormat": "Total Requests", "range": true, "refId": "A" } ], "title": "Incoming Requests", "type": "timeseries" }, { "datasource": { "default": false, "type": "prometheus", "uid": "${datasource}" }, "description": "Rate of requests across the most commonly requested application endpoints.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "req/s" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 16 }, "id": 12, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "12.0.2", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "topk(5, \n sum(\n rate(\n django_http_requests_total_by_view_transport_method_total{\n namespace=\"$namespace\",\n service=\"keystone-api-metrics\",\n method=~\"$method\",\n view=~\"$view\"\n }[$__rate_interval]\n )\n ) by(view) > 0\n)", "instant": false, "legendFormat": "{{method}} {{view}}", "range": true, "refId": "A" } ], "title": "Incoming Requests by View", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "description": "Rate of outgoing responses by response status code.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "resp/s" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 24 }, "id": 37, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "12.0.2", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "sum(\n rate(\n django_http_responses_total_by_status_view_method_total{\n namespace=\"$namespace\",\n service=\"keystone-api-metrics\",\n method=~\"$method\",\n view=~\"$view\"\n }[$__rate_interval]\n )\n) by (status)\n", "instant": false, "legendFormat": "__auto", "range": true, "refId": "A" } ], "title": "Response Status Codes", "type": "timeseries" }, { "datasource": { "default": false, "type": "prometheus", "uid": "${datasource}" }, "description": "Rate of error response codes by application resource.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "text" } ] }, "unit": "err/s" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 24 }, "id": 33, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "12.0.2", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "sum(\n rate(\n django_http_responses_total_by_status_view_method_total{\n namespace=\"$namespace\",\n service=\"keystone-api-metrics\",\n method=~\"$method\",\n view=~\"$view\",\n status=~\"[4-5].+\"\n }[$__rate_interval]\n ) \n) by(view)\n", "instant": false, "legendFormat": "{{view}} ({{status}})", "range": true, "refId": "A" } ], "title": "Response Errors by View", "type": "timeseries" }, { "datasource": { "default": false, "type": "prometheus", "uid": "${datasource}" }, "description": "Time taken to issue a response to incoming requests.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "log": 2, "type": "log" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "s" }, "overrides": [] }, "gridPos": { "h": 9, "w": 12, "x": 0, "y": 32 }, "id": 10, "options": { "legend": { "calcs": [ "mean", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "12.0.2", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "histogram_quantile(0.99, \n sum(\n rate(\n django_http_requests_latency_seconds_by_view_method_bucket{\n namespace=\"$namespace\",\n service=\"keystone-api-metrics\",\n method=~\"$method\",\n view=~\"$view\"\n }[$__rate_interval]\n )\n ) by (le)\n)", "instant": false, "legendFormat": "99% Quantile", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "histogram_quantile(0.95, \n sum(\n rate(\n django_http_requests_latency_seconds_by_view_method_bucket{\n namespace=\"$namespace\",\n service=\"keystone-api-metrics\",\n method=~\"$method\",\n view=~\"$view\"\n }[$__rate_interval]\n )\n ) by (le)\n)", "hide": false, "instant": false, "legendFormat": "95% Quantile", "range": true, "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "histogram_quantile(0.50, \n sum(\n rate(\n django_http_requests_latency_seconds_by_view_method_bucket{\n namespace=\"$namespace\",\n service=\"keystone-api-metrics\",\n method=~\"$method\",\n view=~\"$view\"\n }[$__rate_interval]\n )\n ) by (le)\n)", "hide": false, "instant": false, "legendFormat": "50% Quantile", "range": true, "refId": "C" } ], "title": "Request Latency", "type": "timeseries" }, { "datasource": { "default": false, "type": "prometheus", "uid": "${datasource}" }, "description": "Rate incoming requests by application resource.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "text" } ] }, "unit": "s" }, "overrides": [] }, "gridPos": { "h": 9, "w": 12, "x": 12, "y": 32 }, "id": 34, "options": { "legend": { "calcs": [ "mean", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "12.0.2", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "topk(5, \n avg(\n rate(\n django_http_requests_latency_seconds_by_view_method_bucket{\n namespace=\"$namespace\",\n service=\"keystone-api-metrics\",\n method=~\"$method\",\n view=~\"$view\"\n }[$__rate_interval]\n )\n ) by (view)\n)\n", "instant": false, "legendFormat": "__auto", "range": true, "refId": "A" } ], "title": "Request Latency by View (Avg)", "type": "timeseries" }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 41 }, "id": 25, "panels": [], "title": "Data Throughput", "type": "row" }, { "datasource": { "default": false, "type": "datasource", "uid": "-- Mixed --" }, "description": "Size of the HTTP request body.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "decbytes" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 42 }, "id": 31, "options": { "legend": { "calcs": [ "mean", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true, "sortBy": "Mean", "sortDesc": true }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "desc" } }, "pluginVersion": "12.0.2", "targets": [ { "datasource": { "type": "prometheus", "uid": "$datasource" }, "editorMode": "code", "expr": "histogram_quantile(0.99,\n sum(\n rate(\n django_http_requests_body_total_bytes_bucket{\n namespace=\"$namespace\",\n service=\"keystone-api-metrics\"\n }[$__rate_interval]\n ) > 0\n ) by (le)\n)", "legendFormat": "50% Quantile", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "$datasource" }, "editorMode": "code", "expr": "histogram_quantile(0.95,\n sum(\n irate(\n django_http_requests_body_total_bytes_bucket{\n namespace=\"$namespace\",\n service=\"keystone-api-metrics\"\n }[$__rate_interval]\n ) > 0\n ) by (le)\n)", "legendFormat": "95% Quantile", "range": true, "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "$datasource" }, "editorMode": "code", "expr": "histogram_quantile(0.50,\n sum(\n irate(\n django_http_requests_body_total_bytes_bucket{\n namespace=\"$namespace\",\n service=\"keystone-api-metrics\"\n }[$__rate_interval]\n ) > 0\n ) by (le)\n)\n", "legendFormat": "99% Quantile", "range": true, "refId": "C" } ], "title": "Request Body (All HTTP Methods / Views)", "type": "timeseries" }, { "datasource": { "default": false, "type": "datasource", "uid": "-- Mixed --" }, "description": "Size of the API response body.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "decbytes" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 42 }, "id": 32, "options": { "legend": { "calcs": [ "mean", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true, "sortBy": "Mean", "sortDesc": true }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "desc" } }, "pluginVersion": "12.0.2", "targets": [ { "datasource": { "type": "prometheus", "uid": "$datasource" }, "editorMode": "code", "expr": "histogram_quantile(0.99,\n sum(\n rate(\n django_http_responses_body_total_bytes_bucket{\n namespace=\"$namespace\",\n service=\"keystone-api-metrics\"\n }[$__rate_interval]\n ) > 0\n ) by (le)\n)", "legendFormat": "50% Quantile", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "$datasource" }, "editorMode": "code", "expr": "histogram_quantile(0.95,\n sum(\n irate(\n django_http_responses_body_total_bytes_bucket{\n namespace=\"$namespace\",\n service=\"keystone-api-metrics\"\n }[$__rate_interval]\n ) > 0\n ) by (le)\n)", "legendFormat": "95% Quantile", "range": true, "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "$datasource" }, "editorMode": "code", "expr": "histogram_quantile(0.50,\n sum(\n irate(\n django_http_responses_body_total_bytes_bucket{\n namespace=\"$namespace\",\n service=\"keystone-api-metrics\"\n }[$__rate_interval]\n ) > 0\n ) by (le)\n)\n", "legendFormat": "99% Quantile", "range": true, "refId": "C" } ], "title": "Response Body (All HTTP Methods / Views)", "type": "timeseries" }, { "datasource": { "default": false, "type": "datasource", "uid": "-- Mixed --" }, "description": "Time taken to execute database queries.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "s" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 50 }, "id": 27, "options": { "legend": { "calcs": [ "mean", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true, "sortBy": "Mean", "sortDesc": true }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "desc" } }, "pluginVersion": "12.0.2", "targets": [ { "datasource": { "type": "prometheus", "uid": "$datasource" }, "editorMode": "code", "expr": "histogram_quantile(0.50,\n sum(\n rate(\n django_db_query_duration_seconds_bucket{\n namespace=\"$namespace\",\n service=\"keystone-api-metrics\"\n }[$__rate_interval]\n ) > 0\n ) by (le)\n)", "legendFormat": "50% Quantile", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "$datasource" }, "editorMode": "code", "expr": "histogram_quantile(0.95,\n sum(\n irate(\n django_db_query_duration_seconds_bucket{\n namespace=\"$namespace\",\n service=\"keystone-api-metrics\"\n }[$__rate_interval]\n ) > 0\n ) by (le)\n)", "legendFormat": "95% Quantile", "range": true, "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "$datasource" }, "editorMode": "code", "expr": "histogram_quantile(0.99,\n sum(\n irate(\n django_db_query_duration_seconds_bucket{\n namespace=\"$namespace\",\n service=\"keystone-api-metrics\"\n }[$__rate_interval]\n ) > 0\n ) by (le)\n)\n", "legendFormat": "99% Quantile", "range": true, "refId": "C" } ], "title": "Database Latency", "type": "timeseries" } ], "preload": false, "refresh": "5s", "schemaVersion": 41, "tags": [ "Keystone", "API Server" ], "templating": { "list": [ { "current": { "text": "Prometheus", "value": "PBFA97CFB590B2093" }, "description": "The Prometheus data source.", "includeAll": false, "label": "Datasource", "name": "datasource", "options": [], "query": "prometheus", "refresh": 1, "regex": "", "type": "datasource" }, { "current": { "text": "keystone", "value": "keystone" }, "description": "The Prometheus namespace value to filter data by.", "hide": 2, "label": "Namespace", "name": "namespace", "query": "keystone", "skipUrlSync": true, "type": "constant" }, { "current": { "text": "All", "value": "$__all" }, "datasource": { "type": "prometheus", "uid": "${datasource}" }, "definition": "label_values(django_http_responses_total_by_status_view_method_total,method)", "description": "The HTTP method to query results for.", "includeAll": true, "label": "HTTP Method", "multi": true, "name": "method", "options": [], "query": { "qryType": 1, "query": "label_values(django_http_responses_total_by_status_view_method_total,method)", "refId": "PrometheusVariableQueryEditor-VariableQuery" }, "refresh": 2, "regex": "", "type": "query" }, { "current": { "text": "All", "value": "$__all" }, "datasource": { "type": "prometheus", "uid": "${datasource}" }, "definition": "label_values(django_http_requests_total_by_view_transport_method_total,view)", "description": "Application view responsible for handling the incoming request.", "includeAll": true, "label": "API View", "multi": true, "name": "view", "options": [], "query": { "qryType": 1, "query": "label_values(django_http_requests_total_by_view_transport_method_total,view)", "refId": "PrometheusVariableQueryEditor-VariableQuery" }, "refresh": 1, "regex": "", "type": "query" } ] }, "time": { "from": "now-12h", "to": "now" }, "timepicker": {}, "timezone": "browser", "title": "Keystone API", "uid": "fe3qvo8nrcc8wc", "version": 1 }