增加Exporter功能，可以直接从云厂商获取DRS的CPU、内存、磁盘指标

flask-consul/requirements.txt

@@ -20,8 +20,10 @@ alibabacloud_resourcemanager20200331==2.1.1
 alibabacloud_ecs20140526==2.1.1
 alibabacloud_rds20140815==2.1.1
 alibabacloud_bssopenapi20171214==2.0.5
-tencentcloud-sdk-python-common==3.0.763
+aliyun-python-sdk-cms==7.0.32
-tencentcloud-sdk-python-cvm==3.0.763
+tencentcloud-sdk-python-common==3.0.770
-tencentcloud-sdk-python-cdb==3.0.763
+tencentcloud-sdk-python-cvm==3.0.770
-tencentcloud-sdk-python-dcdb==3.0.763
+tencentcloud-sdk-python-cdb==3.0.770
-tencentcloud-sdk-python-billing==3.0.763
+tencentcloud-sdk-python-dcdb==3.0.770
+tencentcloud-sdk-python-billing==3.0.770
+tencentcloud-sdk-python-monitor==3.0.770

flask-consul/units/gen_config.py

@@ -1,7 +1,23 @@
 from config import consul_token,consul_url
 
-def rds_config(services_list, exporter):
+def rds_config(region_list,cm_exporter,services_list,exporter):
+    region_str = '\n      - '.join([i.replace('/rds','') for i in region_list])
     consul_server = consul_url.split("/")[2]
+    exporter_config = f"""
+  - job_name: 'ConsulManager-exporter'
+    scrape_interval: 30s
+    scrape_timeout: 15s
+    static_configs:
+    - targets:
+      - {region_str}
+    relabel_configs:
+      - source_labels: [__address__]
+        target_label: __metrics_path__
+        regex: (.*)
+        replacement: /api/cloud_mysql_metrics/${{1}}
+      - target_label: __address__
+        replacement: {cm_exporter}
+"""
     configs = f"""
   - job_name: multi_mysqld_exporter
     scrape_interval: 15s
@@ -44,6 +60,16 @@ def rds_config(services_list, exporter):
       - source_labels: ['__meta_consul_service_metadata_itype']
         target_label: itype
 """
+    if not services_list:
+        return {'code': 20000,'configs': '请选择需要Prometheus从Conusl自动发现的MySQL组' }
+    if services_list and exporter == '':
+        return {'code': 20000,'configs': '您已经选择了需要Prometheus从Conusl自动发现MySQL组，\n请输入Mysql_Exporter的地址和端口，例如：10.0.0.26:9104' }
+    if region_list and cm_exporter == '':
+        return {'code': 20000,'configs': '您已经选择了需要从云监控采集基础指标(CPU、内存、磁盘、IO)的MySQL组，\n请输入ConsulManager地址和端口，例如：10.0.0.26:1026' }
+
+    if region_list:
+        return {'code': 20000,'configs': exporter_config + configs }
+    else:
         return {'code': 20000,'configs': configs }
 
 def ecs_config(services_list,ostype_list):

flask-consul/units/prom/mysql_ali.py

@@ -0,0 +1,34 @@
+from aliyunsdkcore.client import AcsClient
+from aliyunsdkcore.acs_exception.exceptions import ClientException
+from aliyunsdkcore.acs_exception.exceptions import ServerException
+from aliyunsdkcms.request.v20190101.DescribeMetricLastRequest import DescribeMetricLastRequest
+from datetime import datetime
+from units import consul_kv
+import json
+
+def exporter(vendor,account,region):
+    ak,sk = consul_kv.get_aksk(vendor,account)
+    client_rdsmonit = AcsClient(ak, sk, region)
+    request_rdsmonit = DescribeMetricLastRequest()
+    request_rdsmonit.set_accept_format('json')
+    request_rdsmonit.set_Namespace("acs_rds_dashboard")
+    metric_name_dict = {"CpuUsage":["# HELP mysql_cpu_util CPU使用率","# TYPE mysql_cpu_util gauge"],
+                        "MemoryUsage":["# HELP mysql_mem_util 内存使用率","# TYPE mysql_mem_util gauge"],
+                        "DiskUsage":["# HELP mysql_disk_util 磁盘使用率","# TYPE mysql_disk_util gauge"],
+                        "IOPSUsage":["# HELP mysql_io_util 磁盘I/O使用率","# TYPE mysql_io_util gauge"]
+                       }
+    try:
+        for i in metric_name_dict.keys():
+            request_rdsmonit.set_MetricName(i)
+            response_rdsmonit = json.loads(client_rdsmonit.do_action_with_exception(request_rdsmonit), encoding='utf-8')
+            instance = json.loads(response_rdsmonit["Datapoints"])
+            prom_metric_name = metric_name_dict[i][0].split()[2]
+            for j in instance:
+                iid,max,ts = j["instanceId"],j["Maximum"],j["timestamp"]
+                metric_name_dict[i].append(f'{prom_metric_name}{{iid="{iid}"}} {float(max)} {ts}')
+        prom_metric_list = []
+        for x in metric_name_dict.values():
+            prom_metric_list = prom_metric_list + x
+        return prom_metric_list
+    except Exception as e:
+        print(e,flush=True)

flask-consul/units/prom/mysql_tencent.py

@@ -0,0 +1,44 @@
+import json
+from tencentcloud.common import credential
+from tencentcloud.common.profile.client_profile import ClientProfile
+from tencentcloud.common.profile.http_profile import HttpProfile
+from tencentcloud.common.exception.tencent_cloud_sdk_exception import TencentCloudSDKException
+from tencentcloud.monitor.v20180724 import monitor_client, models
+from datetime import datetime,timedelta
+from units import consul_kv
+
+def exporter(vendor,account,region):
+    ak,sk = consul_kv.get_aksk(vendor,account)
+    cred = credential.Credential(ak,sk)
+    client = monitor_client.MonitorClient(cred, region)
+    req = models.GetMonitorDataRequest()
+    metric_name_dict = {"CpuUseRate":["# HELP mysql_cpu_util CPU使用率","# TYPE mysql_cpu_util gauge"],
+                        "MemoryUseRate":["# HELP mysql_mem_util 内存使用率","# TYPE mysql_mem_util gauge"],
+                        "IOPS":["# HELP mysql_iops_count 每秒I/O请求数","# TYPE mysql_iops_count gauge"],
+                        "VolumeRate":["# HELP mysql_disk_util 磁盘使用率","# TYPE mysql_disk_util gauge"],
+                        "IopsUseRate":["# HELP mysql_io_util 磁盘I/O使用率","# TYPE mysql_io_util gauge"]
+                       }
+    rds_list = consul_kv.get_services_list_by_region(f'{vendor}_{account}_rds',region)
+    rds_list = list(rds_list)
+    rds_list_10 = [rds_list[i:i + 10] for i in range(0, len(rds_list), 10)]
+    try:
+        for i in metric_name_dict.keys():
+            for rdss in rds_list_10:
+                starttime = (datetime.now() + timedelta(minutes=-1)).strftime('%Y-%m-%dT%H:%M:%S+08:00')
+                ins_list = [{"Dimensions":[{"Name":"InstanceId","Value":x}]} for x in rdss]
+                params = {"Namespace":"QCE/CDB","MetricName":i,"Period":60,"StartTime":starttime,"Instances":ins_list}
+                req.from_json_string(json.dumps(params))
+                resp = client.GetMonitorData(req)
+                metric_list = resp.DataPoints
+                for metrics in metric_list:
+                    iid = metrics.Dimensions[0].Value
+                    value = metrics.Values[-1]
+                    ts = metrics.Timestamps[-1]*1000
+                    prom_metric_name = metric_name_dict[i][0].split()[2]
+                    metric_name_dict[i].append(f'{prom_metric_name}{{iid="{iid}"}} {float(value)} {ts}') 
+        prom_metric_list = []
+        for x in metric_name_dict.values():
+            prom_metric_list = prom_metric_list + x
+        return prom_metric_list
+    except TencentCloudSDKException as err:
+        print(err)

flask-consul/views/nodes.py

@@ -89,7 +89,7 @@ class Nodes(Resource):
         elif stype == 'rdspconfig':
             args = parser.parse_args()
             services_dict = args['services_dict']
-            return gen_config.rds_config(services_dict['services_list'],services_dict['exporter'])
+            return gen_config.rds_config(services_dict['jobrds_list'],services_dict['cm_exporter'],services_dict['services_list'],services_dict['exporter'])
         elif stype == 'cstecs':
             args = parser.parse_args()
             cst_ecs_dict = args['cst_ecs_dict']

flask-consul/views/prom/cloud_mysql_metrics.py

@@ -2,13 +2,18 @@ from flask import Blueprint,Response
 from flask_restful import reqparse, Resource, Api
 from config import vendors,regions
 from units import token_auth,consul_kv
-from units.prom import mysql_huawei
+from units.prom import mysql_huawei,mysql_ali,mysql_tencent
 import json
 blueprint = Blueprint('cloud_mysql_metrics',__name__)
 api = Api(blueprint)
 
 class Exporter(Resource):
     def get(self,vendor,account,region):
+        if vendor == 'huaweicloud':
             prom_metric_list = mysql_huawei.exporter(vendor,account,region)
+        elif vendor == 'alicloud':
+            prom_metric_list = mysql_ali.exporter(vendor,account,region)
+        elif vendor == 'tencent_cloud':
+            prom_metric_list = mysql_tencent.exporter(vendor,account,region)
         return Response('\n'.join(prom_metric_list).encode('utf-8'),mimetype="text/plain")
 api.add_resource(Exporter, '/api/cloud_mysql_metrics/<vendor>/<account>/<region>')

vue-consul/src/main.js

@@ -46,7 +46,7 @@ Object.keys(filters).forEach(key => {
 })
 
 Vue.config.productionTip = false
-Vue.prototype.VER = 'v0.9.4'
+Vue.prototype.VER = 'v0.9.5'
 
 new Vue({
   el: '#app',

vue-consul/src/views/rds/pconfig.vue

@@ -1,10 +1,13 @@
 <template>
   <div class="app-container">
-    <el-select v-model="services" multiple placeholder="请选择需要生成配置的服务" filterable collapse-tags clearable style="width: 350px" class="filter-item">
+    <el-select v-model="services" multiple placeholder="选择需要自动发现的MySQL组" filterable collapse-tags clearable style="width: 260px" class="filter-item">
       <el-option v-for="item in services_list" :key="item" :label="item" :value="item" />
     </el-select>
-    &nbsp;&nbsp;<font color="#ff0000">*</font>MySQLd_Exporter IP端口
+    <el-input v-model="exporter" placeholder="Mysqld_Exporter IP端口" clearable style="width: 200px;" class="filter-item" />&nbsp;&nbsp;
-    <el-input v-model="exporter" placeholder="x.x.x.x:9104" clearable style="width: 200px;" class="filter-item" />&nbsp;&nbsp;
+    <el-select v-model="jobrds" multiple placeholder="选择需要采集指标的MySQL组" filterable collapse-tags clearable style="width: 260px" class="filter-item">
+      <el-option v-for="item in jobrds_list" :key="item" :label="item" :value="item" />
+    </el-select>
+    <el-input v-model="cm_exporter" placeholder="ConsulManager IP端口" clearable style="width: 190px;" class="filter-item" />&nbsp;&nbsp;
     <el-button class="filter-item" type="primary" icon="el-icon-magic-stick" @click="fetchRdsConfig">
       生成配置
     </el-button>
@@ -16,17 +19,20 @@
 </template>
 
 <script>
-import { getRdsServicesList, getRdsConfig } from '@/api/node-exporter'
+import { getRdsServicesList, getRdsConfig, getJobRds } from '@/api/node-exporter'
 export default {
   data() {
     return {
       listLoading: false,
       services: [],
+      jobrds: [],
       ostype: [],
       services_list: [],
       services_dict: {},
+      jobrds_list: [],
       exporter: '',
-      configs: ''
+      cm_exporter: '',
+      configs: '该功能用于生成Prometheus的两个JOB配置，生成后请复制到Prometheus配置中：\n\n1. 选择需要同步的账号，Prometheus即可自动发现该账号下的所有DRS实例。\n\n2. 由于Mysqld_Exporter无法监控到云数据库的CPU、内存、磁盘的使用情况，所以ConsulManager开发了Exporter功能，可以直接从云厂商获取到这些指标并接入Prometheus！\n   选择需要采集指标的RDS账号区域，即可生成Prometheus的JOB配置。'
     }
   },
   created() {
@@ -46,13 +52,18 @@ export default {
       this.listLoading = true
       getRdsServicesList().then(response => {
         this.services_list = response.services_list
-        this.listLoading = false
       })
+      getJobRds().then(response => {
+        this.jobrds_list = response.jobrds
+      })
+      this.listLoading = false
     },
     fetchRdsConfig() {
       this.listLoading = true
       this.services_dict.services_list = this.services
       this.services_dict.exporter = this.exporter
+      this.services_dict.jobrds_list = this.jobrds
+      this.services_dict.cm_exporter = this.cm_exporter
       getRdsConfig(this.services_dict).then(response => {
         this.configs = response.configs
         this.listLoading = false