一、背景

简略记录下,应用PrometheusCanal进行监控。

二、实现步骤

1、批改prometheus.yml配置文件

vim /Users/huan/soft/prometheus/prometheus-2.25.0/prometheus.yml

scrape_configs:  - job_name: 'canal'    scrape_interval: 30s    static_configs:    - targets: ['localhost:11112'] # 端口配置即为canal.properties中的canal.metrics.pull.port      labels:        nodename: 'canal'
# 检测刚刚编写的 prometheus.yml 语法是否谬误./promtool check config prometheus.yml

2、启动prometheus

nohup /Users/huan/soft/prometheus/prometheus-2.25.0/prometheus \--config.file="/Users/huan/soft/prometheus/prometheus-2.25.0/prometheus.yml" \--web.listen-address="0.0.0.0:9080" \--web.enable-lifecycle \--storage.tsdb.retention.time="3d" \--log.level=debug \> logs/prometheus.out 2>&1 &

3、查看prometheus是否胜利接入canal

4、canal原始指标解释

指标阐明单位精度
canal_instance_transactionsinstance接管transactions计数--
canal_instanceinstance根本信息--
canal_instance_subscriptionsinstance订阅数量--
canal_instance_publish_blocking_timeinstance dump线程提交到异步解析队列过程中的阻塞工夫(仅parallel解析模式)msns
canal_instance_received_binlog_bytesinstance接管binlog字节数byte-
canal_instance_parser_modeinstance解析模式(是否开启parallel解析)--
canal_instance_client_packetsinstance client申请次数的计数--
canal_instance_client_bytes向instance client发送数据包字节计数byte-
canal_instance_client_empty_batches向instance client发送get接口的空后果计数--
canal_instance_client_request_errorinstance client申请失败计数--
canal_instance_client_request_latencyinstance client申请的响应工夫详情--
canal_instance_sink_blocking_timeinstance sink线程put数据至store的阻塞工夫msns
canal_instance_store_produce_seqinstance store接管到的events sequence number--
canal_instance_store_consume_seqinstance store胜利生产的events sequence number--
canal_instance_storeinstance store根本信息--
canal_instance_store_produce_meminstance store接管到的所有events占用内存总量byte-
canal_instance_store_consume_meminstance store胜利生产的所有events占用内存总量byte-
canal_instance_put_rowsstore put操作实现的table rows--
canal_instance_get_rowsclient get申请返回的table rows--
canal_instance_ack_rowsclient ack操作开释的table rows--
canal_instance_traffic_delayserver与MySQL master的延时msms
canal_instance_put_delaystore put操作events的延时msms
canal_instance_get_delayclient get申请返回events的延时msms
canal_instance_ack_delayclient ack操作开释events的延时msms

5、导入fana图表

图表json数据

{  "__inputs": [    {      "name": "DS_PROMETHEUS",      "label": "prometheus",      "description": "",      "type": "datasource",      "pluginId": "prometheus",      "pluginName": "Prometheus"    }  ],  "__requires": [    {      "type": "grafana",      "id": "grafana",      "name": "Grafana",      "version": "5.2.2"    },    {      "type": "panel",      "id": "graph",      "name": "Graph",      "version": "5.0.0"    },    {      "type": "datasource",      "id": "prometheus",      "name": "Prometheus",      "version": "5.0.0"    }  ],  "annotations": {    "list": [      {        "builtIn": 1,        "datasource": "-- Grafana --",        "enable": true,        "hide": true,        "iconColor": "rgba(0, 211, 255, 1)",        "name": "Annotations & Alerts",        "type": "dashboard"      }    ]  },  "editable": true,  "gnetId": null,  "graphTooltip": 0,  "id": null,  "iteration": 1536989235272,  "links": [],  "panels": [    {      "collapsed": false,      "gridPos": {        "h": 1,        "w": 24,        "x": 0,        "y": 0      },      "id": 30,      "panels": [],      "title": "Instance status",      "type": "row"    },    {      "aliasColors": {},      "bars": false,      "dashLength": 10,      "dashes": false,      "datasource": "$datasource",      "description": "Canal instance 根本信息。",      "fill": 1,      "gridPos": {        "h": 5,        "w": 6,        "x": 0,        "y": 1      },      "id": 24,      "legend": {        "alignAsTable": true,        "avg": false,        "current": false,        "hideEmpty": false,        "hideZero": false,        "max": false,        "min": false,        "rightSide": true,        "show": true,        "sideWidth": 500,        "total": false,        "values": false      },      "lines": true,      "linewidth": 1,      "links": [],      "nullPointMode": "null",      "percentage": false,      "pointradius": 5,      "points": false,      "renderer": "flot",      "repeat": null,      "seriesOverrides": [],      "spaceLength": 10,      "stack": false,      "steppedLine": false,      "targets": [        {          "expr": "canal_instance{destination=~\"$destination\"}",          "format": "time_series",          "instant": true,          "intervalFactor": 1,          "legendFormat": "Destination: {{destination}}",          "refId": "A"        },        {          "expr": "canal_instance_parser_mode{destination=~\"$destination\"}",          "format": "time_series",          "instant": true,          "intervalFactor": 1,          "legendFormat": "Parallel parser: {{parallel}}",          "refId": "B"        },        {          "expr": "canal_instance_store{destination=~\"$destination\"}",          "format": "time_series",          "instant": true,          "intervalFactor": 1,          "legendFormat": "Batch mode: {{batchMode}}",          "refId": "C"        },        {          "expr": "canal_instance_store{destination=~\"$destination\"}",          "format": "time_series",          "instant": true,          "intervalFactor": 1,          "legendFormat": "Buffer size: {{size}}",          "refId": "D"        }      ],      "thresholds": [],      "timeFrom": null,      "timeShift": null,      "title": "Basic",      "tooltip": {        "shared": true,        "sort": 0,        "value_type": "individual"      },      "transparent": true,      "type": "graph",      "xaxis": {        "buckets": null,        "mode": "time",        "name": null,        "show": false,        "values": []      },      "yaxes": [        {          "format": "short",          "label": null,          "logBase": 1,          "max": null,          "min": null,          "show": false        },        {          "format": "short",          "label": null,          "logBase": 1,          "max": null,          "min": null,          "show": false        }      ],      "yaxis": {        "align": false,        "alignLevel": null      }    },    {      "aliasColors": {        "inbound": "#bf1b00"      },      "bars": false,      "dashLength": 10,      "dashes": false,      "datasource": "$datasource",      "description": "Canal instance 网络带宽占用。\ninbound: 读取MySQL binlog.\noutbound: 对Client端传输格式化binlog.",      "fill": 1,      "gridPos": {        "h": 5,        "w": 6,        "x": 6,        "y": 1      },      "id": 6,      "legend": {        "avg": false,        "current": false,        "max": false,        "min": false,        "show": true,        "total": false,        "values": false      },      "lines": true,      "linewidth": 1,      "links": [],      "nullPointMode": "null",      "percentage": false,      "pointradius": 5,      "points": false,      "renderer": "flot",      "seriesOverrides": [],      "spaceLength": 10,      "stack": false,      "steppedLine": false,      "targets": [        {          "expr": "rate(canal_instance_received_binlog_bytes{destination=~\"$destination\", parser=\"0\"}[2m]) / 1024",          "format": "time_series",          "interval": "15s",          "intervalFactor": 2,          "legendFormat": "inbound",          "refId": "A"        },        {          "expr": "rate(canal_instance_client_bytes{destination=~\"$destination\"}[2m]) / 1024",          "format": "time_series",          "interval": "15s",          "intervalFactor": 2,          "legendFormat": "outbound",          "refId": "B"        },        {          "expr": "rate(canal_instance_received_binlog_bytes{destination=~\"$destination\", parser=\"1\"}[2m]) / 1024",          "format": "time_series",          "interval": "15s",          "intervalFactor": 2,          "legendFormat": "inbound-1",          "refId": "C"        },        {          "expr": "rate(canal_instance_received_binlog_bytes{destination=~\"$destination\", parser=\"2\"}[2m]) / 1024",          "format": "time_series",          "interval": "15s",          "intervalFactor": 2,          "legendFormat": "inbound-2",          "refId": "D"        }      ],      "thresholds": [],      "timeFrom": null,      "timeShift": null,      "title": "Network bandwith",      "tooltip": {        "shared": true,        "sort": 0,        "value_type": "individual"      },      "type": "graph",      "xaxis": {        "buckets": null,        "mode": "time",        "name": null,        "show": true,        "values": []      },      "yaxes": [        {          "format": "KBs",          "label": null,          "logBase": 1,          "max": null,          "min": null,          "show": true        },        {          "format": "short",          "label": null,          "logBase": 1,          "max": null,          "min": null,          "show": true        }      ],      "yaxis": {        "align": false,        "alignLevel": null      }    },    {      "aliasColors": {        "ack": "#f29191",        "get": "#cca300",        "put": "#1f78c1"      },      "bars": false,      "dashLength": 10,      "dashes": false,      "datasource": "$datasource",      "description": "master: Canal server绝对于MySQL master的延时。通过master heartbeat机制能刷新idle状态下的延时。\nput: store put操作的工夫点为基准。\nget: client get操作的工夫点为基准。\nack:  client ack操作的工夫点为基准。",      "fill": 1,      "gridPos": {        "h": 5,        "w": 6,        "x": 12,        "y": 1      },      "id": 4,      "legend": {        "avg": false,        "current": false,        "max": false,        "min": false,        "show": true,        "total": false,        "values": false      },      "lines": true,      "linewidth": 1,      "links": [],      "nullPointMode": "null",      "percentage": false,      "pointradius": 5,      "points": false,      "renderer": "flot",      "seriesOverrides": [],      "spaceLength": 10,      "stack": false,      "steppedLine": false,      "targets": [        {          "expr": "canal_instance_traffic_delay{destination=~\"$destination\"} / 1000",          "format": "time_series",          "hide": false,          "interval": "15s",          "intervalFactor": 2,          "legendFormat": "master",          "refId": "D"        },        {          "expr": "canal_instance_put_delay{destination=~\"$destination\"} / 1000",          "format": "time_series",          "interval": "15s",          "intervalFactor": 2,          "legendFormat": "put",          "refId": "A"        },        {          "expr": "canal_instance_get_delay{destination=~\"$destination\"} / 1000",          "format": "time_series",          "interval": "15s",          "intervalFactor": 2,          "legendFormat": "get",          "refId": "B"        },        {          "expr": "canal_instance_ack_delay{destination=~\"$destination\"} / 1000",          "format": "time_series",          "interval": "15s",          "intervalFactor": 2,          "legendFormat": "ack",          "refId": "C"        }      ],      "thresholds": [],      "timeFrom": null,      "timeShift": null,      "title": "Delay",      "tooltip": {        "shared": true,        "sort": 0,        "value_type": "individual"      },      "type": "graph",      "xaxis": {        "buckets": null,        "mode": "time",        "name": null,        "show": true,        "values": []      },      "yaxes": [        {          "format": "s",          "label": null,          "logBase": 1,          "max": null,          "min": null,          "show": true        },        {          "format": "short",          "label": null,          "logBase": 1,          "max": null,          "min": null,          "show": true        }      ],      "yaxis": {        "align": false,        "alignLevel": null      }    },    {      "aliasColors": {},      "bars": false,      "dashLength": 10,      "dashes": false,      "datasource": "$datasource",      "description": "sink线程blocking占比;dump线程blocking占比(仅parallel mode)。",      "fill": 1,      "gridPos": {        "h": 5,        "w": 6,        "x": 18,        "y": 1      },      "hideTimeOverride": false,      "id": 2,      "legend": {        "avg": false,        "current": false,        "max": false,        "min": false,        "show": true,        "total": false,        "values": false      },      "lines": true,      "linewidth": 1,      "links": [],      "nullPointMode": "null",      "percentage": false,      "pointradius": 5,      "points": false,      "renderer": "flot",      "seriesOverrides": [],      "spaceLength": 10,      "stack": false,      "steppedLine": false,      "targets": [        {          "expr": "clamp_max(rate(canal_instance_publish_blocking_time{destination=~\"$destination\", parser=\"0\"}[2m]), 1000) / 10",          "format": "time_series",          "interval": "15s",          "intervalFactor": 2,          "legendFormat": "dump",          "refId": "B"        },        {          "expr": "clamp_max(rate(canal_instance_sink_blocking_time{destination=~\"$destination\"}[2m]), 1000) / 10",          "format": "time_series",          "interval": "15s",          "intervalFactor": 2,          "legendFormat": "sink",          "refId": "A"        },        {          "expr": "clamp_max(rate(canal_instance_publish_blocking_time{destination=~\"$destination\", parser=\"1\"}[2m]), 1000) / 10",          "format": "time_series",          "interval": "15s",          "intervalFactor": 2,          "legendFormat": "dump-1",          "refId": "C"        },        {          "expr": "clamp_max(rate(canal_instance_publish_blocking_time{destination=~\"$destination\", parser=\"2\"}[2m]), 1000) / 10",          "format": "time_series",          "interval": "15s",          "intervalFactor": 2,          "legendFormat": "dump-2",          "refId": "D"        }      ],      "thresholds": [],      "timeFrom": null,      "timeShift": null,      "title": "Blocking",      "tooltip": {        "shared": true,        "sort": 0,        "value_type": "individual"      },      "type": "graph",      "xaxis": {        "buckets": null,        "mode": "time",        "name": null,        "show": true,        "values": []      },      "yaxes": [        {          "format": "percent",          "label": null,          "logBase": 1,          "max": null,          "min": null,          "show": true        },        {          "format": "short",          "label": null,          "logBase": 1,          "max": null,          "min": null,          "show": true        }      ],      "yaxis": {        "align": false,        "alignLevel": null      }    },    {      "collapsed": false,      "gridPos": {        "h": 1,        "w": 24,        "x": 0,        "y": 6      },      "id": 32,      "panels": [],      "title": "Throughput",      "type": "row"    },    {      "aliasColors": {        "rowDatas": "#7eb26d",        "tableRows": "#c15c17"      },      "bars": false,      "dashLength": 10,      "dashes": false,      "datasource": "$datasource",      "description": "Instance解决binlog的TPS(以master变更行数table rows为基准计算)。\nput: put操作TPS。\nget: get操作TPS。\nack: ack操作TPS。",      "fill": 1,      "gridPos": {        "h": 5,        "w": 6,        "x": 0,        "y": 7      },      "id": 14,      "legend": {        "avg": false,        "current": false,        "max": false,        "min": false,        "show": true,        "total": false,        "values": false      },      "lines": true,      "linewidth": 1,      "links": [],      "nullPointMode": "null",      "percentage": false,      "pointradius": 5,      "points": false,      "renderer": "flot",      "seriesOverrides": [],      "spaceLength": 10,      "stack": false,      "steppedLine": false,      "targets": [        {          "expr": "rate(canal_instance_put_rows{destination=~\"$destination\"}[2m])",          "format": "time_series",          "interval": "15s",          "intervalFactor": 2,          "legendFormat": "put",          "refId": "A"        },        {          "expr": "rate(canal_instance_get_rows{destination=~\"$destination\"}[2m])",          "format": "time_series",          "interval": "15s",          "intervalFactor": 2,          "legendFormat": "get",          "refId": "B"        },        {          "expr": "rate(canal_instance_ack_rows{destination=~\"$destination\"}[2m])",          "format": "time_series",          "intervalFactor": 1,          "legendFormat": "ack",          "refId": "C"        }      ],      "thresholds": [],      "timeFrom": null,      "timeShift": null,      "title": "TPS(table rows)",      "tooltip": {        "shared": true,        "sort": 0,        "value_type": "individual"      },      "type": "graph",      "xaxis": {        "buckets": null,        "mode": "time",        "name": null,        "show": true,        "values": []      },      "yaxes": [        {          "format": "iops",          "label": null,          "logBase": 1,          "max": null,          "min": null,          "show": true        },        {          "format": "short",          "label": null,          "logBase": 1,          "max": null,          "min": null,          "show": true        }      ],      "yaxis": {        "align": false,        "alignLevel": null      }    },    {      "aliasColors": {        "transactions": "#f9ba8f"      },      "bars": false,      "dashLength": 10,      "dashes": false,      "datasource": "$datasource",      "description": "Canal instance 解决binlog的TPS,以MySQL transaction为单位计算。",      "fill": 1,      "gridPos": {        "h": 5,        "w": 6,        "x": 6,        "y": 7      },      "id": 12,      "legend": {        "avg": false,        "current": false,        "max": false,        "min": false,        "show": true,        "total": false,        "values": false      },      "lines": true,      "linewidth": 1,      "links": [],      "nullPointMode": "null",      "percentage": false,      "pointradius": 5,      "points": false,      "renderer": "flot",      "seriesOverrides": [],      "spaceLength": 10,      "stack": false,      "steppedLine": false,      "targets": [        {          "expr": "rate(canal_instance_transactions{destination=~\"$destination\"}[2m])",          "format": "time_series",          "interval": "15s",          "intervalFactor": 2,          "legendFormat": "transactions",          "refId": "A"        }      ],      "thresholds": [],      "timeFrom": null,      "timeShift": null,      "title": "TPS(MySQL transaction)",      "tooltip": {        "shared": true,        "sort": 0,        "value_type": "individual"      },      "type": "graph",      "xaxis": {        "buckets": null,        "mode": "time",        "name": null,        "show": true,        "values": []      },      "yaxes": [        {          "format": "iops",          "label": null,          "logBase": 1,          "max": null,          "min": null,          "show": true        },        {          "format": "short",          "label": null,          "logBase": 1,          "max": null,          "min": null,          "show": true        }      ],      "yaxis": {        "align": false,        "alignLevel": null      }    },    {      "collapsed": false,      "gridPos": {        "h": 1,        "w": 24,        "x": 0,        "y": 12      },      "id": 34,      "panels": [],      "title": "Client",      "type": "row"    },    {      "aliasColors": {},      "bars": false,      "dashLength": 10,      "dashes": false,      "datasource": "$datasource",      "description": "Canal instance接管到的申请统计,后果按packet type分类。",      "fill": 1,      "gridPos": {        "h": 5,        "w": 6,        "x": 0,        "y": 13      },      "id": 16,      "legend": {        "avg": false,        "current": false,        "max": false,        "min": false,        "show": true,        "total": false,        "values": false      },      "lines": true,      "linewidth": 1,      "links": [],      "nullPointMode": "null",      "percentage": false,      "pointradius": 5,      "points": false,      "renderer": "flot",      "seriesOverrides": [],      "spaceLength": 10,      "stack": false,      "steppedLine": false,      "targets": [        {          "expr": "canal_instance_client_packets{destination=~\"$destination\"}",          "format": "time_series",          "interval": "15s",          "intervalFactor": 2,          "legendFormat": "{{packetType}}",          "refId": "A"        }      ],      "thresholds": [],      "timeFrom": null,      "timeShift": null,      "title": "Client requests",      "tooltip": {        "shared": true,        "sort": 0,        "value_type": "individual"      },      "type": "graph",      "xaxis": {        "buckets": null,        "mode": "time",        "name": null,        "show": true,        "values": []      },      "yaxes": [        {          "format": "none",          "label": null,          "logBase": 1,          "max": null,          "min": null,          "show": true        },        {          "format": "short",          "label": null,          "logBase": 1,          "max": null,          "min": null,          "show": true        }      ],      "yaxis": {        "align": false,        "alignLevel": null      }    },    {      "aliasColors": {},      "bars": false,      "dashLength": 10,      "dashes": false,      "datasource": "$datasource",      "description": "client 申请的GET与ACK包的QPS。",      "fill": 1,      "gridPos": {        "h": 5,        "w": 6,        "x": 6,        "y": 13      },      "id": 38,      "legend": {        "avg": false,        "current": false,        "max": false,        "min": false,        "show": true,        "total": false,        "values": false      },      "lines": true,      "linewidth": 1,      "links": [],      "nullPointMode": "null",      "percentage": false,      "pointradius": 5,      "points": false,      "renderer": "flot",      "seriesOverrides": [],      "spaceLength": 10,      "stack": false,      "steppedLine": false,      "targets": [        {          "expr": "rate(canal_instance_client_packets{destination=~\"$destination\",packetType=\"GET\"}[2m])",          "format": "time_series",          "interval": "15s",          "intervalFactor": 2,          "legendFormat": "GET",          "refId": "A"        },        {          "expr": "rate(canal_instance_client_packets{destination=~\"$destination\",packetType=\"CLIENTACK\"}[2m])",          "format": "time_series",          "interval": "15s",          "intervalFactor": 2,          "legendFormat": "ACK",          "refId": "B"        }      ],      "thresholds": [],      "timeFrom": null,      "timeShift": null,      "title": "Client QPS",      "tooltip": {        "shared": true,        "sort": 0,        "value_type": "individual"      },      "type": "graph",      "xaxis": {        "buckets": null,        "mode": "time",        "name": null,        "show": true,        "values": []      },      "yaxes": [        {          "format": "short",          "label": null,          "logBase": 1,          "max": null,          "min": null,          "show": true        },        {          "format": "short",          "label": null,          "logBase": 1,          "max": null,          "min": null,          "show": true        }      ],      "yaxis": {        "align": false,        "alignLevel": null      }    },    {      "aliasColors": {},      "bars": false,      "dashLength": 10,      "dashes": false,      "datasource": "$datasource",      "description": "server响应GET申请,但返回空包的占比。",      "fill": 1,      "gridPos": {        "h": 5,        "w": 6,        "x": 12,        "y": 13      },      "id": 26,      "legend": {        "avg": false,        "current": false,        "max": false,        "min": false,        "show": true,        "total": false,        "values": false      },      "lines": true,      "linewidth": 1,      "links": [],      "nullPointMode": "null",      "percentage": false,      "pointradius": 5,      "points": false,      "renderer": "flot",      "seriesOverrides": [],      "spaceLength": 10,      "stack": false,      "steppedLine": false,      "targets": [        {          "expr": "rate(canal_instance_client_empty_batches{destination=~\"$destination\"}[2m])",          "format": "time_series",          "interval": "15s",          "intervalFactor": 2,          "legendFormat": "empty",          "refId": "A"        },        {          "expr": "rate(canal_instance_client_packets{destination=~\"$destination\", packetType=\"GET\"}[2m])",          "format": "time_series",          "intervalFactor": 1,          "legendFormat": "nonempty",          "refId": "B"        }      ],      "thresholds": [],      "timeFrom": null,      "timeShift": null,      "title": "Empty packets",      "tooltip": {        "shared": true,        "sort": 0,        "value_type": "individual"      },      "type": "graph",      "xaxis": {        "buckets": null,        "mode": "time",        "name": null,        "show": true,        "values": []      },      "yaxes": [        {          "format": "wps",          "label": null,          "logBase": 1,          "max": null,          "min": null,          "show": true        },        {          "format": "short",          "label": null,          "logBase": 1,          "max": null,          "min": null,          "show": true        }      ],      "yaxis": {        "align": false,        "alignLevel": null      }    },    {      "aliasColors": {},      "bars": false,      "dashLength": 10,      "dashes": false,      "datasource": "$datasource",      "description": "Canal client 申请响应工夫的详情。",      "fill": 1,      "gridPos": {        "h": 5,        "w": 6,        "x": 18,        "y": 13      },      "id": 18,      "legend": {        "alignAsTable": false,        "avg": false,        "current": false,        "max": false,        "min": false,        "rightSide": false,        "show": true,        "total": false,        "values": false      },      "lines": true,      "linewidth": 1,      "links": [],      "nullPointMode": "null",      "percentage": false,      "pointradius": 5,      "points": false,      "renderer": "flot",      "seriesOverrides": [        {          "alias": "25.0",          "yaxis": 1        },        {          "alias": "100.0",          "yaxis": 1        }      ],      "spaceLength": 10,      "stack": false,      "steppedLine": false,      "targets": [        {          "expr": "rate(canal_instance_client_request_latency_bucket{destination=~\"$destination\"}[2m])",          "format": "time_series",          "hide": false,          "interval": "15s",          "intervalFactor": 2,          "legendFormat": "{{le}}ms",          "refId": "A"        }      ],      "thresholds": [],      "timeFrom": null,      "timeShift": null,      "title": "Response time",      "tooltip": {        "shared": true,        "sort": 0,        "value_type": "individual"      },      "transparent": false,      "type": "graph",      "xaxis": {        "buckets": null,        "mode": "time",        "name": null,        "show": true,        "values": []      },      "yaxes": [        {          "format": "short",          "label": null,          "logBase": 1,          "max": null,          "min": null,          "show": true        },        {          "format": "short",          "label": null,          "logBase": 1,          "max": null,          "min": null,          "show": true        }      ],      "yaxis": {        "align": false,        "alignLevel": null      }    },    {      "collapsed": false,      "gridPos": {        "h": 1,        "w": 24,        "x": 0,        "y": 18      },      "id": 36,      "panels": [],      "title": "Store",      "type": "row"    },    {      "aliasColors": {},      "bars": false,      "dashLength": 10,      "dashes": false,      "datasource": "$datasource",      "description": "Canal instance ringbuffer内未开释的events数量。",      "fill": 1,      "gridPos": {        "h": 5,        "w": 6,        "x": 0,        "y": 19      },      "id": 20,      "legend": {        "avg": false,        "current": false,        "max": false,        "min": false,        "show": true,        "total": false,        "values": false      },      "lines": true,      "linewidth": 1,      "links": [],      "nullPointMode": "null",      "percentage": false,      "pointradius": 5,      "points": false,      "renderer": "flot",      "seriesOverrides": [],      "spaceLength": 10,      "stack": false,      "steppedLine": false,      "targets": [        {          "expr": "canal_instance_store_produce_seq{destination=~\"$destination\"} - canal_instance_store_consume_seq{destination=~\"$destination\"}",          "format": "time_series",          "interval": "15s",          "intervalFactor": 2,          "legendFormat": "events",          "refId": "A"        }      ],      "thresholds": [],      "timeFrom": null,      "timeShift": null,      "title": "Store remain events",      "tooltip": {        "shared": true,        "sort": 0,        "value_type": "individual"      },      "type": "graph",      "xaxis": {        "buckets": null,        "mode": "time",        "name": null,        "show": true,        "values": []      },      "yaxes": [        {          "decimals": null,          "format": "none",          "label": "",          "logBase": 1,          "max": null,          "min": null,          "show": true        },        {          "decimals": null,          "format": "short",          "label": null,          "logBase": 1,          "max": null,          "min": null,          "show": true        }      ],      "yaxis": {        "align": false,        "alignLevel": null      }    },    {      "aliasColors": {},      "bars": false,      "dashLength": 10,      "dashes": false,      "datasource": "$datasource",      "description": "Canal instance ringbuffer 内未开释events占用内存。",      "fill": 1,      "gridPos": {        "h": 5,        "w": 6,        "x": 6,        "y": 19      },      "id": 22,      "legend": {        "avg": false,        "current": false,        "max": false,        "min": false,        "show": true,        "total": false,        "values": false      },      "lines": true,      "linewidth": 1,      "links": [],      "nullPointMode": "null",      "percentage": false,      "pointradius": 5,      "points": false,      "renderer": "flot",      "seriesOverrides": [],      "spaceLength": 10,      "stack": false,      "steppedLine": false,      "targets": [        {          "expr": "(canal_instance_store_produce_mem{destination=~\"$destination\"} - canal_instance_store_consume_mem{destination=~\"$destination\"}) / 1024",          "format": "time_series",          "interval": "15s",          "intervalFactor": 2,          "legendFormat": "memsize",          "refId": "A"        }      ],      "thresholds": [],      "timeFrom": null,      "timeShift": null,      "title": "Store remain mem",      "tooltip": {        "shared": true,        "sort": 0,        "value_type": "individual"      },      "type": "graph",      "xaxis": {        "buckets": null,        "mode": "time",        "name": null,        "show": true,        "values": []      },      "yaxes": [        {          "format": "deckbytes",          "label": null,          "logBase": 1,          "max": null,          "min": null,          "show": true        },        {          "format": "short",          "label": null,          "logBase": 1,          "max": null,          "min": null,          "show": true        }      ],      "yaxis": {        "align": false,        "alignLevel": null      }    }  ],  "refresh": false,  "schemaVersion": 16,  "style": "dark",  "tags": [    "canal"  ],  "templating": {    "list": [      {        "current": {          "text": "prometheus",          "value": "prometheus"        },        "hide": 0,        "label": "datasource",        "name": "datasource",        "options": [],        "query": "prometheus",        "refresh": 1,        "regex": "",        "type": "datasource"      },      {        "allValue": null,        "current": {},        "datasource": "$datasource",        "hide": 0,        "includeAll": false,        "label": "destination",        "multi": false,        "name": "destination",        "options": [],        "query": "label_values(canal_instance, destination)",        "refresh": 1,        "regex": "",        "sort": 0,        "tagValuesQuery": "",        "tags": [],        "tagsQuery": "",        "type": "query",        "useTags": false      }    ]  },  "time": {    "from": "now-6h",    "to": "now"  },  "timepicker": {    "refresh_intervals": [      "5s",      "10s",      "30s",      "1m",      "5m",      "15m",      "30m",      "1h",      "2h",      "1d"    ],    "time_options": [      "5m",      "15m",      "1h",      "6h",      "12h",      "24h",      "2d",      "7d",      "30d"    ]  },  "timezone": "",  "title": "Canal instances",  "uid": "8vh8NGpiz",  "version": 103}

三、参考链接

1、https://github.com/alibaba/canal/wiki/Prometheus-QuickStart