{ "__inputs": [], "__requires": [ { "type": "grafana", "id": "grafana", "name": "Grafana", "version": "7.5.11" }, { "type": "panel", "id": "graph", "name": "Graph", "version": "" }, { "type": "datasource", "id": "prometheus", "name": "Prometheus", "version": "1.0.0" } ], "annotations": { "list": [ { "builtIn": 1, "datasource": "-- Grafana --", "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", "name": "Annotations & Alerts", "type": "dashboard" } ] }, "editable": true, "gnetId": null, "graphTooltip": 0, "id": null, "iteration": 1639752893375, "links": [], "panels": [ { "collapsed": true, "datasource": null, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, "id": 4, "panels": [ { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$source", "fieldConfig": { "defaults": { "custom": {}, "links": [] }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 1 }, "hiddenSeries": false, "id": 2, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.3.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "avg(elasticsearch_cluster_health_number_of_nodes{cluster=\"$cluster\"})", "interval": "", "legendFormat": "total nodes", "refId": "A" }, { "expr": "avg(elasticsearch_cluster_health_number_of_data_nodes{cluster=\"$cluster\"})", "interval": "", "legendFormat": "data nodes", "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Nodes", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": "Elasticsearch nodes count", "logBase": 1, "max": null, "min": "0", "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$source", "fieldConfig": { "defaults": { "custom": {}, "links": [] }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 1 }, "hiddenSeries": false, "id": 5, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.3.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [ { "alias": "/%/", "dashes": true, "lines": false, "yaxis": 2 } ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "avg(elasticsearch_cluster_health_active_shards{cluster=\"$cluster\"})", "interval": "", "legendFormat": "Active shards", "refId": "A" }, { "expr": "avg(elasticsearch_cluster_health_active_primary_shards{cluster=\"$cluster\"})", "interval": "", "legendFormat": "Primary shards", "refId": "B" }, { "expr": " min(elasticsearch_cluster_health_active_shards_percent_as_number{cluster=\"$cluster\"})", "interval": "", "legendFormat": "% Active shards", "refId": "C" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Shards", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": "Elasticsearch shards count", "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": 0, "format": "percent", "label": "Percent of active shards", "logBase": 1, "max": "100", "min": "0", "show": true } ], "yaxis": { "align": false, "alignLevel": null } } ], "title": "Cluster health: /_cluster/health", "type": "row" }, { "collapsed": false, "datasource": null, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 1 }, "id": 7, "panels": [], "title": "Thread pool main: /_cat/thread_pool", "type": "row" }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$source", "description": "Number of tasks in the queue for the current thread pool.\n\nHigh queue indicates that requests are \"waiting\" in line to be executed. In best circumstances queue should be close to zero. Elasticsearch imposes limit on the queue capacity, after capacity is reached, requests in queue will be rejected.\n\nDocs: https://www.elastic.co/guide/en/elasticsearch/reference/current/cat-thread-pool.html#cat-thread-pool-query-params", "fieldConfig": { "defaults": { "links": [] }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 2 }, "hiddenSeries": false, "id": 14, "interval": "", "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.11", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [ { "$$hashKey": "object:169", "alias": "/write/", "dashes": true, "lines": false } ], "spaceLength": 10, "stack": false, "steppedLine": true, "targets": [ { "expr": "sum(max(increase(elasticsearch_cat_thread_pool_queue{cluster=\"$cluster\"}[$time])) by (node_name, name)) by (name)", "hide": false, "interval": "", "intervalFactor": 10, "legendFormat": "{{name}}", "refId": "A" }, { "expr": "max(elasticsearch_cat_thread_pool_queue{cluster=\"$cluster\", name=~\"search\", node_name=~\"$thread_pool_queue_search\"}) by (node_name)", "hide": false, "interval": "", "intervalFactor": 10, "legendFormat": "search: {{node_name}}", "refId": "B" }, { "expr": "max(elasticsearch_cat_thread_pool_queue{cluster=\"$cluster\", name=~\"write\", node_name=~\"$thread_pool_queue_write\"}) by (node_name)", "hide": false, "interval": "", "intervalFactor": 10, "legendFormat": "write: {{node_name}}", "refId": "C" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Thread pool: queue", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:173", "decimals": null, "format": "short", "label": "Threads count", "logBase": 1, "max": null, "min": "0", "show": true }, { "$$hashKey": "object:174", "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$source", "description": "Number of active threads in the current thread pool.\n\nActive threads \"working\" in Elasticsearch cluster. Graph show total main threads \"search\" and \"write\" compared to currently active threads.\n\nDocs: https://www.elastic.co/guide/en/elasticsearch/reference/current/cat-thread-pool.html#cat-thread-pool-query-params", "fieldConfig": { "defaults": { "links": [] }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 6, "x": 12, "y": 2 }, "hiddenSeries": false, "id": 8, "interval": "$min_interval", "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.11", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [ { "alias": "/Total/", "dashes": true, "lines": false } ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(max(elasticsearch_cat_thread_pool_active{cluster=\"$cluster\"}) by (node_name, name)) by (name)", "hide": false, "instant": false, "interval": "", "intervalFactor": 4, "legendFormat": "{{name}}", "refId": "A" }, { "expr": "sum(max(elasticsearch_cat_thread_pool_pool_size{cluster=\"$cluster\", name=~\"write\"}) by (node_name))", "hide": false, "instant": false, "interval": "", "intervalFactor": 4, "legendFormat": "Total: write", "refId": "B" }, { "expr": "sum(max(elasticsearch_cat_thread_pool_pool_size{cluster=\"$cluster\", name=~\"search\"}) by (node_name))", "instant": false, "interval": "", "intervalFactor": 4, "legendFormat": "Total: search", "refId": "C" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Thread pool: active", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "short", "label": "Active threads in pool", "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$source", "description": "Number of rejected threads in the current thread pool.\n\nRejected threads happen when cluster is \"overloaded\" usually when Elasticsearch thread pool queue capacity is reached. When cluster queue capacity is reached Elasticsearch will start rejecting requests to protect cluster from complete outage.\n\nDocs: https://www.elastic.co/guide/en/elasticsearch/reference/current/cat-thread-pool.html#cat-thread-pool-query-params", "fieldConfig": { "defaults": { "links": [] }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 6, "x": 18, "y": 2 }, "hiddenSeries": false, "id": 77, "interval": "$min_interval", "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.11", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(increase(elasticsearch_cat_thread_pool_rejected{cluster=\"$cluster\"}[$time])) by (name)", "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{name}}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Thread pool: rejected", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "short", "label": "Active threads in pool", "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "collapsed": true, "datasource": null, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 10 }, "id": 16, "panels": [ { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$source", "description": "Configured fixed number of active threads allowed in the current thread pool.\n\nDocs: https://www.elastic.co/guide/en/elasticsearch/reference/current/cat-thread-pool.html#cat-thread-pool-query-params", "fieldConfig": { "defaults": { "custom": {}, "links": [] }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 6, "x": 0, "y": 11 }, "hiddenSeries": false, "id": 13, "interval": "$min_interval", "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.3.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(max(elasticsearch_cat_thread_pool_pool_size{cluster=\"$cluster\"}) by (node_name, name)) by (name) ", "interval": "", "intervalFactor": 2, "legendFormat": "{{name}}", "refId": "A" }, { "expr": "sum(max(elasticsearch_cat_thread_pool_pool_size{cluster=\"$cluster\"}) by (node_name, name))", "interval": "", "intervalFactor": 2, "legendFormat": "Total size", "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Thread pool: size", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "short", "label": "Threads count per pool", "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$source", "description": "Configured core number of active threads allowed in the current thread pool.\n\nDocs: https://www.elastic.co/guide/en/elasticsearch/reference/current/cat-thread-pool.html#cat-thread-pool-query-params", "fieldConfig": { "defaults": { "custom": {}, "links": [] }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 6, "x": 6, "y": 11 }, "hiddenSeries": false, "id": 9, "interval": "$min_interval", "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.3.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(max(elasticsearch_cat_thread_pool_core{cluster=\"$cluster\"}) by (node_name, name)) by (name) ", "hide": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{name}}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Thread pool: core", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "short", "label": "Threads count", "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$source", "description": "Highest number of active threads in the current thread pool.\n\nDocs: https://www.elastic.co/guide/en/elasticsearch/reference/current/cat-thread-pool.html#cat-thread-pool-query-params", "fieldConfig": { "defaults": { "custom": {}, "links": [] }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 6, "x": 12, "y": 11 }, "hiddenSeries": false, "id": 12, "interval": "$min_interval", "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.3.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(max(elasticsearch_cat_thread_pool_largest{cluster=\"$cluster\"}) by (node_name, name)) by (name) ", "hide": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{name}}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Thread pool: largest", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "short", "label": "Threads count", "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$source", "description": "Number of completed tasks\n\nCompleted thread pool \"tasks\" is useful information about thread trends. This graph indicates current throughput of per thread type in operations per minute.\n\nDocs: https://www.elastic.co/guide/en/elasticsearch/reference/current/modules-threadpool.html#scaling-thread-pool", "fieldConfig": { "defaults": { "custom": {}, "links": [] }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 6, "x": 18, "y": 11 }, "hiddenSeries": false, "id": 11, "interval": "$min_interval", "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.3.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(max(irate(elasticsearch_cat_thread_pool_completed{cluster=\"$cluster\"}[$time])) by (node_name, name)) by (name) ", "hide": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{name}}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Thread pool: completed", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "opm", "label": "Tasks", "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$source", "description": "Number of tasks in the queue for the current thread pool.\n\nCurrent queue size in Elasticsearch cluster per thread type.\n\nDocs: https://www.elastic.co/guide/en/elasticsearch/reference/current/cat-thread-pool.html#cat-thread-pool-query-params", "fieldConfig": { "defaults": { "custom": {}, "links": [] }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 6, "x": 0, "y": 19 }, "hiddenSeries": false, "id": 78, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.3.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": true, "targets": [ { "expr": "sum(max(elasticsearch_cat_thread_pool_queue_size{cluster=\"$cluster\"}) by (node_name, name)) by (name)", "interval": "", "intervalFactor": 10, "legendFormat": "{{name}}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Thread pool: queue size", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "short", "label": "Threads count", "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$source", "description": "Configured maximum number of active threads allowed in the current thread pool.\n\nMaximum number of recorded threads per type in Elasticsearch cluster.\n\nDocs: https://www.elastic.co/guide/en/elasticsearch/reference/current/cat-thread-pool.html#cat-thread-pool-query-params", "fieldConfig": { "defaults": { "custom": {}, "links": [] }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 6, "x": 6, "y": 19 }, "hiddenSeries": false, "id": 10, "interval": "$min_interval", "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.3.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(max(elasticsearch_cat_thread_pool_max{cluster=\"$cluster\"}) by (node_name, name)) by (name) ", "interval": "", "intervalFactor": 2, "legendFormat": "{{name}}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Thread pool: max", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "short", "label": "Threads count", "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } } ], "title": "Thread pool limits: /_cat/thread_pool", "type": "row" }, { "collapsed": true, "datasource": null, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 11 }, "id": 74, "panels": [ { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$source", "description": "Request cache hit ratio calculated by hits / (hits + miss). Cache hit ratio is expressed in percentage.\n\nDocs: https://www.elastic.co/guide/en/elasticsearch/reference/current/cat-indices.html", "fieldConfig": { "defaults": { "custom": {}, "links": [] }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 6, "x": 0, "y": 4 }, "hiddenSeries": false, "id": 60, "interval": "$min_interval", "legend": { "alignAsTable": true, "avg": true, "current": true, "max": true, "min": false, "show": true, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "nullPointMode": "null as zero", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.3.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "min(increase(elasticsearch_cat_indices_request_cache_hit_count{cluster=\"$cluster\", index=~\"$index\"}[$time]) / (increase(elasticsearch_cat_indices_request_cache_hit_count{cluster=\"$cluster\", index=~\"$index\"}[$time]) + increase(elasticsearch_cat_indices_request_cache_miss_count{cluster=\"$cluster\", index=~\"$index\"}[$time]))) by (index)", "hide": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{index}}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Request cache hit ratio", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": 1, "format": "percentunit", "label": "Hit ratio", "logBase": 1, "max": "1", "min": "0", "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$source", "description": "Used request cache memory in Elasticsearch cluster.\n\nDocs: https://www.elastic.co/guide/en/elasticsearch/reference/current/cat-indices.html", "fieldConfig": { "defaults": { "custom": {}, "links": [] }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 6, "x": 6, "y": 4 }, "hiddenSeries": false, "id": 57, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.3.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "max(elasticsearch_cat_indices_request_cache_memory_size{cluster=\"$cluster\", index=~\"$index\"}) by (index)", "interval": "", "intervalFactor": 2, "legendFormat": "{{index}}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Request cache memory", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "bytes", "label": "Size in bytes", "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$source", "description": "Request cache evictions. Cluster might not have evictions if refresh interval happens frequently and/or cluster has enough memory to cache all requests before starting evicting the cache.\n\nDocs: https://www.elastic.co/guide/en/elasticsearch/reference/current/cat-indices.html", "fieldConfig": { "defaults": { "custom": {}, "links": [] }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 6, "x": 12, "y": 4 }, "hiddenSeries": false, "id": 76, "interval": "$min_interval", "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.3.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "max(elasticsearch_cat_indices_request_cache_evictions{cluster=\"$cluster\", index=~\"$index\"}) by (index)", "hide": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{index}}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Request cache evictions", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "opm", "label": "Count", "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$source", "description": "Request cache hit count\n\nDocs: https://www.elastic.co/guide/en/elasticsearch/reference/current/cat-indices.html", "fieldConfig": { "defaults": { "custom": {}, "links": [] }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 3, "x": 18, "y": 4 }, "hiddenSeries": false, "id": 58, "interval": "$min_interval", "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.3.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "max(increase(elasticsearch_cat_indices_request_cache_hit_count{cluster=\"$cluster\", index=~\"$index\"}[$time])) by (index)", "hide": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{index}}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Request cache hit", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "opm", "label": "Count", "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$source", "description": "Request cache miss count\n\nDocs: https://www.elastic.co/guide/en/elasticsearch/reference/current/cat-indices.html", "fieldConfig": { "defaults": { "custom": {}, "links": [] }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 3, "x": 21, "y": 4 }, "hiddenSeries": false, "id": 59, "interval": "$min_interval", "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.3.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "max(increase(elasticsearch_cat_indices_request_cache_miss_count{cluster=\"$cluster\", index=~\"$index\"}[$time])) by (index)", "hide": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{index}}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Request cache miss", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "opm", "label": "Count", "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$source", "description": "Used query cache memory in Elasticsearch cluster.\n\nDocs: https://www.elastic.co/guide/en/elasticsearch/reference/current/cat-indices.html", "fieldConfig": { "defaults": { "custom": {}, "links": [] }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 6, "x": 6, "y": 12 }, "hiddenSeries": false, "id": 75, "interval": "$min_interval", "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.3.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "max(elasticsearch_cat_indices_query_cache_memory_size{cluster=\"$cluster\", index=~\"$index\"}) by (index)", "interval": "", "intervalFactor": 2, "legendFormat": "{{index}}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Query cache memory", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "bytes", "label": "Size in bytes", "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$source", "description": "Query cache evictions. Cluster might not have evictions if refresh interval happens frequently and/or cluster has enough memory to cache all requests before starting evicting the cache.\n\nDocs: https://www.elastic.co/guide/en/elasticsearch/reference/current/cat-indices.html", "fieldConfig": { "defaults": { "custom": {}, "links": [] }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 6, "x": 12, "y": 12 }, "hiddenSeries": false, "id": 79, "interval": "$min_interval", "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.3.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "max(increase(elasticsearch_cat_indices_query_cache_evictions{cluster=\"$cluster\", index=~\"$index\"}[$time])) by (index)", "hide": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{index}}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Query cache evictions", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "opm", "label": "Count", "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } } ], "title": "Indices: /_cat/nodes cache", "type": "row" }, { "collapsed": true, "datasource": null, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 12 }, "id": 48, "panels": [ { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$source", "description": "Current query phase operations per second.\n\nDocs: https://www.elastic.co/guide/en/elasticsearch/reference/current/cat-indices.html", "fieldConfig": { "defaults": { "custom": {}, "links": [] }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 6, "x": 0, "y": 5 }, "hiddenSeries": false, "id": 55, "interval": "", "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.3.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "max(elasticsearch_cat_indices_search_query_current{cluster=\"$cluster\", index=~\"$index\"}) by (index)", "hide": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{index}}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Index query: current", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": "Operations", "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$source", "description": "Current fetch phase operations per second.\n\nDocs: https://www.elastic.co/guide/en/elasticsearch/reference/current/cat-indices.html", "fieldConfig": { "defaults": { "custom": {}, "links": [] }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 6, "x": 6, "y": 5 }, "hiddenSeries": false, "id": 56, "interval": "", "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.3.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "max(elasticsearch_cat_indices_search_fetch_current{cluster=\"$cluster\", index=~\"$index\"}) by (index)", "hide": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{index}}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Index fetch: current", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": "Documents", "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$source", "description": "Time spent in refresh operations in Elasticsearch cluster.\n\nDocs: https://www.elastic.co/guide/en/elasticsearch/reference/current/cat-indices.html", "fieldConfig": { "defaults": { "custom": {}, "links": [] }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 6, "x": 12, "y": 5 }, "hiddenSeries": false, "id": 50, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null as zero", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.3.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "max(increase(elasticsearch_cat_indices_refresh_time_seconds{cluster=\"$cluster\", index=~\"$index\"}[$time]) / increase(elasticsearch_cat_indices_indexing_index_total{cluster=\"$cluster\", index=~\"$index\"}[$time])) by (index)", "hide": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{index}}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Index refreshing time", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "s", "label": "Time spent", "logBase": 1, "max": null, "min": "0", "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$source", "description": "Time spent in query phase in Elasticsearch cluster.\n\nDocs: https://www.elastic.co/guide/en/elasticsearch/reference/current/cat-indices.html", "fieldConfig": { "defaults": { "custom": {}, "links": [] }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 6, "x": 0, "y": 13 }, "hiddenSeries": false, "id": 52, "interval": "", "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.3.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "max(increase(elasticsearch_cat_indices_search_query_time_seconds{cluster=\"$cluster\", index=~\"$index\"}[$time]) / increase(elasticsearch_cat_indices_search_query_total{cluster=\"$cluster\", index=~\"$index\"}[$time])) by (index)", "hide": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{index}}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Index query time", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "s", "label": "Time spent", "logBase": 1, "max": null, "min": "0", "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$source", "description": "Time spent in fetch phase in Elasticsearch cluster.\n\nDocs: https://www.elastic.co/guide/en/elasticsearch/reference/current/cat-indices.html", "fieldConfig": { "defaults": { "custom": {}, "links": [] }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 6, "x": 6, "y": 13 }, "hiddenSeries": false, "id": 54, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null as zero", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.3.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "max(increase(elasticsearch_cat_indices_search_fetch_time_seconds{cluster=\"$cluster\", index=~\"$index\"}[$time]) / increase(elasticsearch_cat_indices_search_fetch_total{cluster=\"$cluster\", index=~\"$index\"}[$time])) by (index)", "hide": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{index}}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Index fetch time", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "s", "label": "Time spent", "logBase": 1, "max": null, "min": "0", "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$source", "description": "Number of total refresh counts in Elasticsearch cluster.\n\nDocs: https://www.elastic.co/guide/en/elasticsearch/reference/current/cat-indices.html", "fieldConfig": { "defaults": { "custom": {}, "links": [] }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 6, "x": 12, "y": 13 }, "hiddenSeries": false, "id": 49, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.3.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "max(increase(elasticsearch_cat_indices_refresh_total{cluster=\"$cluster\", index=~\"$index\"}[$time])) by (index)", "hide": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{index}}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Index total refreshes", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "short", "label": "Documents", "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$source", "description": "Total query phase operations per second.\n\nDocs: https://www.elastic.co/guide/en/elasticsearch/reference/current/cat-indices.html", "fieldConfig": { "defaults": { "custom": {}, "links": [] }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 6, "x": 0, "y": 21 }, "hiddenSeries": false, "id": 51, "interval": "$min_interval", "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.3.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "max(increase(elasticsearch_cat_indices_search_query_total{cluster=\"$cluster\", index=~\"$index\"}[$time])) by (index)", "hide": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{index}}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Index total query", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "short", "label": "Documents", "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$source", "description": "Total fetch operations per second in Elasticsearch cluster.\n\nDocs: https://www.elastic.co/guide/en/elasticsearch/reference/current/cat-indices.html", "fieldConfig": { "defaults": { "custom": {}, "links": [] }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 6, "x": 6, "y": 21 }, "hiddenSeries": false, "id": 53, "interval": "$min_interval", "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.3.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "max(increase(elasticsearch_cat_indices_search_fetch_total{cluster=\"$cluster\", index=~\"$index\"}[$time])) by (index)", "hide": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{index}}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Index total fetch", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "short", "label": "Documents", "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } } ], "title": "Indices: /_cat/indices index query, fetch refresh", "type": "row" }, { "collapsed": true, "datasource": null, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 13 }, "id": 18, "panels": [ { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$source", "description": "Available document in the Elasticsearch cluster.\n\nDocs: https://www.elastic.co/guide/en/elasticsearch/reference/current/cat-indices.html", "fieldConfig": { "defaults": { "custom": {}, "links": [] }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 6, "x": 0, "y": 6 }, "hiddenSeries": false, "id": 20, "interval": "$min_interval", "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.3.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "max(elasticsearch_cat_indices_docs_count{cluster=\"$cluster\", index=~\"$index\"}) by (index)", "interval": "", "intervalFactor": 2, "legendFormat": "{{index}}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Total documents", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": "Documents", "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$source", "description": "Deleted documents in Elasticsearch cluster.\n\nDocs: https://www.elastic.co/guide/en/elasticsearch/reference/current/cat-indices.html", "fieldConfig": { "defaults": { "custom": {}, "links": [] }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 6, "x": 6, "y": 6 }, "hiddenSeries": false, "id": 21, "interval": "$min_interval", "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.3.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "max(increase(elasticsearch_cat_indices_docs_deleted{cluster=\"$cluster\", index=~\"$index\"}[$time])) by (index)", "hide": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{index}}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Deleted documents", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": "Documents", "logBase": 1, "max": null, "min": "0", "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$source", "description": "Total time spent performing flush operations. Flush is operation of writing data back to disk.\n\nDocs: https://www.elastic.co/guide/en/elasticsearch/reference/current/cat-indices.html", "fieldConfig": { "defaults": { "custom": {}, "links": [] }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 6, "x": 12, "y": 6 }, "hiddenSeries": false, "id": 25, "interval": "$min_interval", "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null as zero", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.3.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "max(increase(elasticsearch_cat_indices_flush_total_time_seconds{cluster=\"$cluster\", index=~\"$index\"}[$time]) / increase(elasticsearch_cat_indices_flush_total{cluster=\"$cluster\", index=~\"$index\"}[$time])) by (index)", "hide": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{index}}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Flushed document flush time", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "s", "label": "Time spent", "logBase": 1, "max": null, "min": "0", "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$source", "description": "The field data cache contains field data and global ordinals, which are both used to support aggregations on certain field types. Since these are on-heap data structures, it is important to monitor the cache’s use.\n\nDocs: https://www.elastic.co/guide/en/elasticsearch/reference/master/modules-fielddata.html", "fieldConfig": { "defaults": { "custom": {}, "links": [] }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 6, "x": 18, "y": 6 }, "hiddenSeries": false, "id": 22, "interval": "$min_interval", "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.3.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "max(elasticsearch_cat_indices_fielddata_memory_size{cluster=\"$cluster\"}) by (index)", "hide": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{index}}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Field data size", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "bytes", "label": "Size in bytes", "logBase": 1, "max": null, "min": "0", "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$source", "description": "Number of times index was flushed. Flush is operation of writing data back to disk.\n\nDocs: https://www.elastic.co/guide/en/elasticsearch/reference/current/cat-indices.html", "fieldConfig": { "defaults": { "custom": {}, "links": [] }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 6, "x": 12, "y": 14 }, "hiddenSeries": false, "id": 23, "interval": "$min_interval", "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null as zero", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.3.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "max(increase(elasticsearch_cat_indices_flush_total{cluster=\"$cluster\", index=~\"$index\"}[$time])) by (index)", "hide": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{index}}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Number of flushes", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "short", "label": "Documents", "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } } ], "title": "Indices: /_cat/indices documents, flushes & field data", "type": "row" }, { "collapsed": true, "datasource": null, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 14 }, "id": 46, "panels": [ { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$source", "description": "Total index primary shards per index in Elasticsearch cluster.\n\nDocs: https://www.elastic.co/guide/en/elasticsearch/reference/current/cat-indices.html", "fieldConfig": { "defaults": { "custom": {}, "links": [] }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 6, "x": 0, "y": 7 }, "hiddenSeries": false, "id": 44, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.3.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "max(elasticsearch_cat_indices_pri{cluster=\"$cluster\"}) by (index)", "interval": "", "legendFormat": "{{index}}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Index primaries", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "short", "label": "Count", "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$source", "description": "Total index replica shards per index in Elasticsearch cluster.\n\nDocs: https://www.elastic.co/guide/en/elasticsearch/reference/current/cat-indices.html", "fieldConfig": { "defaults": { "custom": {}, "links": [] }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 6, "x": 6, "y": 7 }, "hiddenSeries": false, "id": 42, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.3.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "max(elasticsearch_cat_indices_rep{cluster=\"$cluster\"}) by (index)", "interval": "", "legendFormat": "{{index}}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Index Replicas", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "short", "label": "Count", "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$source", "description": "Open search contexts in Elasticsearch cluster. Search context is the state that is maintained for the duration of a search operation on a shard. The more concurrent search operations you have the more search context objects are active at any one time.\n\nDocs: https://www.elastic.co/guide/en/elasticsearch/reference/current/cat-indices.html", "fieldConfig": { "defaults": { "custom": {}, "links": [] }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 6, "x": 12, "y": 7 }, "hiddenSeries": false, "id": 72, "interval": "$min_interval", "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.3.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "max(elasticsearch_cat_indices_search_open_contexts{cluster=\"$cluster\", index=~\"$index\"}) by (index)", "interval": "", "intervalFactor": 2, "legendFormat": "{{index}}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Open contexts", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "short", "label": "Count", "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } } ], "title": "Indices: /_cat/indices settings & misc", "type": "row" }, { "collapsed": true, "datasource": null, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 15 }, "id": 28, "panels": [ { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$source", "description": "Time spent in indexing operations.\n\nDocs: https://www.elastic.co/guide/en/elasticsearch/reference/current/cat-indices.html", "fieldConfig": { "defaults": { "custom": {}, "links": [] }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 6, "x": 0, "y": 8 }, "hiddenSeries": false, "id": 33, "interval": "$min_interval", "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.3.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "max(increase(elasticsearch_cat_indices_indexing_index_time_seconds{cluster=\"$cluster\", index=~\"$index\"}[$time]) / increase(elasticsearch_cat_indices_indexing_index_total{cluster=\"$cluster\", index=~\"$index\"}[$time])) by (index)", "hide": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{index}}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Index indexing time", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "s", "label": "Time spent", "logBase": 1, "max": null, "min": "0", "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$source", "description": "Number of current indexing operations per second in Elasticsearch cluster.\n\nDocs: https://www.elastic.co/guide/en/elasticsearch/reference/current/cat-indices.html", "fieldConfig": { "defaults": { "custom": {}, "links": [] }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 6, "x": 6, "y": 8 }, "hiddenSeries": false, "id": 32, "interval": "$min_interval", "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.3.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "max(elasticsearch_cat_indices_indexing_index_current{cluster=\"$cluster\", index=~\"$index\"}) by (index)", "hide": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{index}}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Indexing operations: current", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "opm", "label": "Operations", "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$source", "description": "Number of failed indexing operations per second in Elasticsearch cluster.\n\nDocs: https://www.elastic.co/guide/en/elasticsearch/reference/current/cat-indices.html", "fieldConfig": { "defaults": { "custom": {}, "links": [] }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 6, "x": 12, "y": 8 }, "hiddenSeries": false, "id": 31, "interval": "$min_interval", "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.3.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "max(rate(elasticsearch_cat_indices_indexing_index_failed{cluster=\"$cluster\", index=~\"$index\"}[$time])) by (index)", "hide": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{index}}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Indexing failed documents", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "short", "label": "Documents", "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$source", "description": "Time spent to delete documents in seconds. Deleted documents are not removed instantly. Deleted documents are deleted after cluster refresh happens on each index shard per index in Elasticsearch cluster.\n\nDocs: https://www.elastic.co/guide/en/elasticsearch/reference/current/cat-indices.html", "fieldConfig": { "defaults": { "custom": {}, "links": [] }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 6, "x": 18, "y": 8 }, "hiddenSeries": false, "id": 29, "interval": "$min_interval", "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null as zero", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.3.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "max(increase(elasticsearch_cat_indices_indexing_delete_time_seconds{cluster=\"$cluster\", index=~\"$index\"}[$time]) / increase(elasticsearch_cat_indices_indexing_delete_total{cluster=\"$cluster\", index=~\"$index\"}[$time])) by (index)", "hide": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{index}}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Deleted document deletion time", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "s", "label": "Time spent", "logBase": 1, "max": null, "min": "0", "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$source", "description": "Number of indexed documents per index in Elasticsearch cluster.\n\nDocs: https://www.elastic.co/guide/en/elasticsearch/reference/current/cat-indices.html", "fieldConfig": { "defaults": { "custom": {}, "links": [] }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 6, "x": 0, "y": 16 }, "hiddenSeries": false, "id": 34, "interval": "$min_interval", "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.3.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "max(increase(elasticsearch_cat_indices_indexing_index_total{cluster=\"$cluster\", index=~\"$index\"}[$time])) by (index)", "hide": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{index}}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Indexing total documents", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "short", "label": "Documents", "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$source", "description": "Number of current deletions in Elasticsearch cluster.\n\nDocs: https://www.elastic.co/guide/en/elasticsearch/reference/current/cat-indices.html", "fieldConfig": { "defaults": { "custom": {}, "links": [] }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 6, "x": 18, "y": 16 }, "hiddenSeries": false, "id": 26, "interval": "$min_interval", "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.3.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "max(increase(elasticsearch_cat_indices_indexing_delete_total{cluster=\"$cluster\", index=~\"$index\"}[$time])) by (index)", "hide": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{index}}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Indexing deleted documents", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": "Documents", "logBase": 1, "max": null, "min": "0", "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } } ], "title": "Indices: /_cat/indices indexing", "type": "row" }, { "collapsed": true, "datasource": null, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 16 }, "id": 36, "panels": [ { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$source", "description": "Total used memory for indexing operations per index in Elasticsearch cluster.\n\nDocs: https://www.elastic.co/guide/en/elasticsearch/reference/current/cat-indices.html", "fieldConfig": { "defaults": { "custom": {}, "links": [] }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 6, "x": 0, "y": 9 }, "hiddenSeries": false, "id": 38, "interval": "$min_interval", "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null as zero", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.3.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "max(increase(elasticsearch_cat_indices_memory_total{cluster=\"$cluster\", index=~\"$index\"}[$time])) by (index)", "interval": "", "intervalFactor": 2, "legendFormat": "{{index}}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Index memory total", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "bytes", "label": "Size in bytes", "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$source", "description": "Number of current merges per index in Elasticsearch cluster.\n\nDocs: https://www.elastic.co/guide/en/elasticsearch/reference/current/cat-indices.html", "fieldConfig": { "defaults": { "custom": {}, "links": [] }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 6, "x": 6, "y": 9 }, "hiddenSeries": false, "id": 39, "interval": "$min_interval", "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.3.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "max(elasticsearch_cat_indices_merges_current{cluster=\"$cluster\", index=~\"$index\"}) by (index)", "hide": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{index}}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Index merges: current", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "short", "label": "Operations", "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$source", "description": "Number of completed merge operations per second per index in Elasticsearch cluster.\n\nDocs: https://www.elastic.co/guide/en/elasticsearch/reference/current/cat-indices.html", "fieldConfig": { "defaults": { "custom": {}, "links": [] }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 6, "x": 12, "y": 9 }, "hiddenSeries": false, "id": 40, "interval": "$min_interval", "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.3.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "max(increase(elasticsearch_cat_indices_merges_total{cluster=\"$cluster\", index=~\"$index\"}[$time])) by (index)", "hide": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{index}}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Merges total", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "opm", "label": "Operations", "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "short", "label": "", "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$source", "description": "Size merged of documents per index in Elasticsearch cluster.\n\nDocs: https://www.elastic.co/guide/en/elasticsearch/reference/current/cat-indices.html", "fieldConfig": { "defaults": { "custom": {}, "links": [] }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 6, "x": 18, "y": 9 }, "hiddenSeries": false, "id": 41, "interval": "$min_interval", "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null as zero", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.3.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "max(rate(elasticsearch_cat_indices_merges_total_size{cluster=\"$cluster\", index=~\"$index\"}[$time])) by (index)", "interval": "", "intervalFactor": 2, "legendFormat": "{{index}}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Merged total size", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "bytes", "label": "Size in bytes", "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } } ], "title": "Indices: /_cat/indices memory & merging", "type": "row" }, { "collapsed": true, "datasource": null, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 17 }, "id": 62, "panels": [ { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$source", "description": "Number of total segments per index in Elasticsearch cluster.\n\nDocs: https://www.elastic.co/guide/en/elasticsearch/reference/current/cat-indices.html", "fieldConfig": { "defaults": { "custom": {}, "links": [] }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 6, "x": 0, "y": 10 }, "hiddenSeries": false, "id": 63, "interval": "$min_interval", "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.3.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "max(elasticsearch_cat_indices_segments_count{cluster=\"$cluster\", index=~\"$index\"}) by (index)", "interval": "", "intervalFactor": 2, "legendFormat": "{{index}}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Index segments count", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": "Count", "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$source", "description": "Time spent in warmers per index. Some data in the index needs to be loaded into memory for fast access to the index. Since this loading is sometimes lazy, warmers are run just before a new segment gets published in order to ensure that following requests won't need to load this data as part of the query execution. The two main things that warmers load into memory are norms (used for scoring) and field data (used for sorting and facets/aggregations).\n\nDocs: https://www.elastic.co/guide/en/elasticsearch/reference/current/cat-indices.html", "fieldConfig": { "defaults": { "custom": {}, "links": [] }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 6, "x": 6, "y": 10 }, "hiddenSeries": false, "id": 71, "interval": "$min_interval", "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.3.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "max(increase(elasticsearch_cat_indices_warmer_total_time_seconds{cluster=\"$cluster\", index=~\"$index\"}[$time]) / increase(elasticsearch_cat_indices_warmer_total{cluster=\"$cluster\"}[$time])) by (index)", "hide": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{index}}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Warmer time", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "s", "label": "Time spent", "logBase": 1, "max": null, "min": "0", "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$source", "description": "Total memory used by segments per index. \nEach shard contains multiple \"segments\", where a segment is an inverted index. A search in a shard will search each segment in turn, then combine their results into the final results for that shard.\n\nWhile you are indexing documents, Elasticsearch collects them in memory (and in the transaction log, for safety) then every second or so, writes a new small segment to disk, and \"refreshes\" the search.\n\nThis makes the data in the new segment visible to search (ie they are \"searchable\"), but the segment has not been fsync'ed to disk, so is still at risk of data loss.\n\nEvery so often, Elasticsearch will \"flush\", which means fsync'ing the segments, (they are now \"committed\") and clearing out the transaction log, which is no longer needed because we know that the new data has been written to disk.\n\nThe more segments there are, the longer each search takes. So Elasticsearch will merge a number of segments of a similar size (\"tier\") into a single bigger segment, through a background merge process. Once the new bigger segment is written, the old segments are dropped. This process is repeated on the bigger segments when there are too many of the same size.\n\nSegments are immutable. When a document is updated, it actually just marks the old document as deleted, and indexes a new document. The merge process also expunges these old deleted documents.\n\nDocs: https://www.elastic.co/guide/en/elasticsearch/reference/current/cat-indices.html", "fieldConfig": { "defaults": { "custom": {}, "links": [] }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 6, "x": 12, "y": 10 }, "hiddenSeries": false, "id": 64, "interval": "$min_interval", "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.3.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "max(elasticsearch_cat_indices_segments_memory_bytes{cluster=\"$cluster\", index=~\"$index\"}) by (index)", "interval": "", "intervalFactor": 2, "legendFormat": "{{index}}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Segments memory", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "bytes", "label": "Size in bytes", "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$source", "description": "Total memory used by version map per index. Each shard contains multiple \"segments\", where a segment is an inverted index. A search in a shard will search each segment in turn, then combine their results into the final results for that shard.\n\nWhile you are indexing documents, Elasticsearch collects them in memory (and in the transaction log, for safety) then every second or so, writes a new small segment to disk, and \"refreshes\" the search.\n\nThis makes the data in the new segment visible to search (ie they are \"searchable\"), but the segment has not been fsync'ed to disk, so is still at risk of data loss.\n\nEvery so often, Elasticsearch will \"flush\", which means fsync'ing the segments, (they are now \"committed\") and clearing out the transaction log, which is no longer needed because we know that the new data has been written to disk.\n\nThe more segments there are, the longer each search takes. So Elasticsearch will merge a number of segments of a similar size (\"tier\") into a single bigger segment, through a background merge process. Once the new bigger segment is written, the old segments are dropped. This process is repeated on the bigger segments when there are too many of the same size.\n\nSegments are immutable. When a document is updated, it actually just marks the old document as deleted, and indexes a new document. The merge process also expunges these old deleted documents.\n\nDocs: https://www.elastic.co/guide/en/elasticsearch/reference/current/cat-indices.html", "fieldConfig": { "defaults": { "custom": {}, "links": [] }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 6, "x": 18, "y": 10 }, "hiddenSeries": false, "id": 67, "interval": "$min_interval", "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.3.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "max(elasticsearch_cat_indices_segments_version_map_memory_bytes{cluster=\"$cluster\", index=~\"$index\"}) by (index)", "interval": "", "intervalFactor": 2, "legendFormat": "{{index}}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Segments version map memory", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "bytes", "label": "Size in bytes", "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$source", "description": "Total warmer operations in second. Some data in the index needs to be loaded into memory for fast access to the index. Since this loading is sometimes lazy, warmers are run just before a new segment gets published in order to ensure that following requests won't need to load this data as part of the query execution. The two main things that warmers load into memory are norms (used for scoring) and field data (used for sorting and facets/aggregations).\n\nDocs: https://www.elastic.co/guide/en/elasticsearch/reference/current/cat-indices.html", "fieldConfig": { "defaults": { "custom": {}, "links": [] }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 6, "x": 6, "y": 18 }, "hiddenSeries": false, "id": 70, "interval": "$min_interval", "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.3.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "max(increase(elasticsearch_cat_indices_warmer_total{cluster=\"$cluster\", index=~\"$index\"}[$time])) by (index)", "interval": "", "intervalFactor": 2, "legendFormat": "{{index}}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Warmer total", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": "Count", "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$source", "description": "Total storage size of primaries & replicas per index in Elasticsearch cluster.\n\nDocs: https://www.elastic.co/guide/en/elasticsearch/reference/current/cat-indices.html", "fieldConfig": { "defaults": { "custom": {}, "links": [] }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 6, "x": 12, "y": 18 }, "hiddenSeries": false, "id": 68, "interval": "", "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.3.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "max(elasticsearch_cat_indices_store_size{cluster=\"$cluster\", index=~\"$index\"}) by (index)", "interval": "", "intervalFactor": 10, "legendFormat": "{{index}}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Store size", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "bytes", "label": "Size in bytes", "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$source", "description": "Memory used by fixed bit sets per index. Each shard contains multiple \"segments\", where a segment is an inverted index. A search in a shard will search each segment in turn, then combine their results into the final results for that shard.\n\nWhile you are indexing documents, Elasticsearch collects them in memory (and in the transaction log, for safety) then every second or so, writes a new small segment to disk, and \"refreshes\" the search.\n\nThis makes the data in the new segment visible to search (ie they are \"searchable\"), but the segment has not been fsync'ed to disk, so is still at risk of data loss.\n\nEvery so often, Elasticsearch will \"flush\", which means fsync'ing the segments, (they are now \"committed\") and clearing out the transaction log, which is no longer needed because we know that the new data has been written to disk.\n\nThe more segments there are, the longer each search takes. So Elasticsearch will merge a number of segments of a similar size (\"tier\") into a single bigger segment, through a background merge process. Once the new bigger segment is written, the old segments are dropped. This process is repeated on the bigger segments when there are too many of the same size.\n\nSegments are immutable. When a document is updated, it actually just marks the old document as deleted, and indexes a new document. The merge process also expunges these old deleted documents.\n\nDocs: https://www.elastic.co/guide/en/elasticsearch/reference/current/cat-indices.html", "fieldConfig": { "defaults": { "custom": {}, "links": [] }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 6, "x": 18, "y": 18 }, "hiddenSeries": false, "id": 66, "interval": "$min_interval", "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.3.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "max(elasticsearch_cat_indices_segments_fixed_bitset_memory_bytes{cluster=\"$cluster\", index=~\"$index\"}) by (index)", "interval": "", "intervalFactor": 2, "legendFormat": "{{index}}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Segments fixed bitset memory", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "bytes", "label": "Size in bytes", "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "$source", "description": "Memory used by index writer per index. Each shard contains multiple \"segments\", where a segment is an inverted index. A search in a shard will search each segment in turn, then combine their results into the final results for that shard.\n\nWhile you are indexing documents, Elasticsearch collects them in memory (and in the transaction log, for safety) then every second or so, writes a new small segment to disk, and \"refreshes\" the search.\n\nThis makes the data in the new segment visible to search (ie they are \"searchable\"), but the segment has not been fsync'ed to disk, so is still at risk of data loss.\n\nEvery so often, Elasticsearch will \"flush\", which means fsync'ing the segments, (they are now \"committed\") and clearing out the transaction log, which is no longer needed because we know that the new data has been written to disk.\n\nThe more segments there are, the longer each search takes. So Elasticsearch will merge a number of segments of a similar size (\"tier\") into a single bigger segment, through a background merge process. Once the new bigger segment is written, the old segments are dropped. This process is repeated on the bigger segments when there are too many of the same size.\n\nSegments are immutable. When a document is updated, it actually just marks the old document as deleted, and indexes a new document. The merge process also expunges these old deleted documents.\n\nDocs: https://www.elastic.co/guide/en/elasticsearch/reference/current/cat-indices.html", "fieldConfig": { "defaults": { "custom": {}, "links": [] }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 6, "x": 18, "y": 26 }, "hiddenSeries": false, "id": 65, "interval": "$min_interval", "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.3.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "max(elasticsearch_cat_indices_segments_index_writer_memory_bytes{cluster=\"$cluster\", index=~\"$index\"}) by (index)", "interval": "", "intervalFactor": 2, "legendFormat": "{{index}}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Segments index writer memory", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "bytes", "label": "Size in bytes", "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } } ], "title": "Indices: /_cat/indices segments, warmer & store", "type": "row" } ], "refresh": false, "schemaVersion": 27, "style": "dark", "tags": [ "elasticsearch", "cpu", "cache", "indices", "nodes", "system", "thread pool" ], "templating": { "list": [ { "current": { "selected": false, "text": "monitoring-prometheus", "value": "monitoring-prometheus" }, "description": null, "error": null, "hide": 0, "includeAll": false, "label": "", "multi": false, "name": "source", "options": [], "query": "prometheus", "queryValue": "", "refresh": 1, "regex": "", "skipUrlSync": false, "type": "datasource" }, { "allValue": null, "current": {}, "datasource": "$source", "definition": "query_result(count(elasticsearch_cluster_health_number_of_nodes) by (cluster))", "description": null, "error": null, "hide": 0, "includeAll": false, "label": null, "multi": false, "name": "cluster", "options": [], "query": { "query": "query_result(count(elasticsearch_cluster_health_number_of_nodes) by (cluster))", "refId": "monitoring-prometheus-cluster-Variable-Query" }, "refresh": 2, "regex": ".*cluster=\"(.+)\".*", "skipUrlSync": false, "sort": 5, "tagValuesQuery": "", "tags": [], "tagsQuery": "", "type": "query", "useTags": false }, { "allValue": null, "current": { "selected": true, "text": "5m", "value": "5m" }, "description": null, "error": null, "hide": 0, "includeAll": false, "label": null, "multi": false, "name": "time", "options": [ { "selected": false, "text": "1m", "value": "1m" }, { "selected": true, "text": "5m", "value": "5m" }, { "selected": false, "text": "10m", "value": "10m" }, { "selected": false, "text": "30m", "value": "30m" }, { "selected": false, "text": "1h", "value": "1h" }, { "selected": false, "text": "6h", "value": "6h" }, { "selected": false, "text": "12h", "value": "12h" }, { "selected": false, "text": "1d", "value": "1d" }, { "selected": false, "text": "7d", "value": "7d" }, { "selected": false, "text": "14d", "value": "14d" }, { "selected": false, "text": "30d", "value": "30d" } ], "query": "1m,5m,10m,30m,1h,6h,12h,1d,7d,14d,30d", "queryValue": "", "skipUrlSync": false, "type": "custom" }, { "allValue": null, "current": {}, "datasource": "$source", "definition": "query_result(increase(elasticsearch_cat_indices_segments_memory_bytes{cluster=\"$cluster\"}[${__range_s}s]) > 0)", "description": null, "error": null, "hide": 0, "includeAll": true, "label": null, "multi": true, "name": "index", "options": [], "query": { "query": "query_result(increase(elasticsearch_cat_indices_segments_memory_bytes{cluster=\"$cluster\"}[${__range_s}s]) > 0)", "refId": "monitoring-prometheus-index-Variable-Query" }, "refresh": 2, "regex": ".*index=\"(.+?)\".*", "skipUrlSync": false, "sort": 1, "tagValuesQuery": "", "tags": [], "tagsQuery": "", "type": "query", "useTags": false }, { "allValue": null, "current": {}, "datasource": "$source", "definition": "query_result(topk($topk,sum(increase(elasticsearch_cat_thread_pool_queue{cluster=\"$cluster\", name=~\"search\"}[${__range_s}s])) by (node_name) ))", "description": null, "error": null, "hide": 2, "includeAll": true, "label": null, "multi": true, "name": "thread_pool_queue_search", "options": [], "query": { "query": "query_result(topk($topk,sum(increase(elasticsearch_cat_thread_pool_queue{cluster=\"$cluster\", name=~\"search\"}[${__range_s}s])) by (node_name) ))", "refId": "monitoring-prometheus-thread_pool_queue_search-Variable-Query" }, "refresh": 2, "regex": ".*name=\"(.+?)\".*", "skipUrlSync": false, "sort": 1, "tagValuesQuery": "", "tags": [], "tagsQuery": "", "type": "query", "useTags": false }, { "allValue": null, "current": { "selected": false, "text": "3", "value": "3" }, "description": null, "error": null, "hide": 0, "includeAll": false, "label": null, "multi": false, "name": "topk", "options": [ { "selected": true, "text": "3", "value": "3" }, { "selected": false, "text": "5", "value": "5" }, { "selected": false, "text": "10", "value": "10" }, { "selected": false, "text": "20", "value": "20" } ], "query": "3,5,10,20", "queryValue": "", "skipUrlSync": false, "type": "custom" }, { "allValue": null, "current": {}, "datasource": "$source", "definition": "query_result(topk($topk,sum(increase(elasticsearch_cat_thread_pool_queue{cluster=\"$cluster\", name=~\"write\"}[${__range_s}s])) by (node_name) ))", "description": null, "error": null, "hide": 2, "includeAll": true, "label": null, "multi": true, "name": "thread_pool_queue_write", "options": [], "query": { "query": "query_result(topk($topk,sum(increase(elasticsearch_cat_thread_pool_queue{cluster=\"$cluster\", name=~\"write\"}[${__range_s}s])) by (node_name) ))", "refId": "monitoring-prometheus-thread_pool_queue_write-Variable-Query" }, "refresh": 2, "regex": ".*name=\"(.+?)\".*", "skipUrlSync": false, "sort": 1, "tagValuesQuery": "", "tags": [], "tagsQuery": "", "type": "query", "useTags": false }, { "allValue": null, "current": { "selected": false, "text": "30s", "value": "30s" }, "description": null, "error": null, "hide": 0, "includeAll": false, "label": null, "multi": false, "name": "min_interval", "options": [ { "selected": false, "text": "15s", "value": "15s" }, { "selected": true, "text": "30s", "value": "30s" }, { "selected": false, "text": "45s", "value": "45s" }, { "selected": false, "text": "60s", "value": "60s" } ], "query": "15s,30s,45s,60s", "queryValue": "", "skipUrlSync": false, "type": "custom" } ] }, "time": { "from": "now-12h", "to": "now" }, "timepicker": { "refresh_intervals": [ "10s", "30s", "1m", "5m", "15m", "30m", "1h", "2h", "1d" ] }, "timezone": "utc", "title": "Elasticsearch Exporter: general", "uid": "WsgCfeTMz", "version": 156 }