增加采集华为云mysql的各类资源使用率，可作为prometheus的exporter使用

flask-consul/manager.py

@@ -8,6 +8,7 @@ if consul_kv.get_kv_dict(skey_path) == {}:
     consul_kv.put_kv(skey_path,{'sk':''.join(str(uuid.uuid4()).split('-'))})
 
 from views import login, blackbox, consul, jobs, nodes, selfnode, avd, exp, jms, edit_cloud
+from views.prom import cloud_mysql_metrics
 from units.cloud import huaweicloud,alicloud,tencent_cloud
 from units.avd import avd_list
 from units.jms import sync_jms
@@ -26,6 +27,7 @@ app.register_blueprint(avd.blueprint)
 app.register_blueprint(exp.blueprint)
 app.register_blueprint(jms.blueprint)
 app.register_blueprint(edit_cloud.blueprint)
+app.register_blueprint(cloud_mysql_metrics.blueprint)
 
 class Config(object):
     JOBS = []

flask-consul/requirements.txt

@@ -10,11 +10,12 @@ xlrd==1.2.0
 #pyDes==2.0.1
 pycryptodome==3.14.1
 beautifulsoup4==4.11.1
-huaweicloudsdkcore==3.1.5
-huaweicloudsdkecs==3.1.5
-huaweicloudsdkeps==3.1.5
-huaweicloudsdkbss==3.1.5
-huaweicloudsdkrds==3.1.5
+huaweicloudsdkcore==3.1.8
+huaweicloudsdkecs==3.1.8
+huaweicloudsdkeps==3.1.8
+huaweicloudsdkbss==3.1.8
+huaweicloudsdkrds==3.1.8
+huaweicloudsdkces==3.1.8
 alibabacloud_resourcemanager20200331==2.1.1
 alibabacloud_ecs20140526==2.1.1
 alibabacloud_rds20140815==2.1.1

flask-consul/units/cloud/alicloud.py

@@ -19,7 +19,7 @@ from units.cloud import sync_rds
 from units.cloud import notify
 
 def exp(account,collect_days,notify_days,notify_amount):
-    print(f"=====【阿里云：余额与到期日统计开始：{account}】", flush=True)
+    #print(f"=====【阿里云：余额与到期日统计开始：{account}】", flush=True)
     ak,sk = consul_kv.get_aksk('alicloud',account)
     now = datetime.datetime.utcnow().strftime('%Y-%m-%dT16:00:00Z')
     collect = (datetime.datetime.utcnow() + datetime.timedelta(days=collect_days+1)).strftime('%Y-%m-%dT16:00:00Z')
@@ -36,7 +36,7 @@ def exp(account,collect_days,notify_days,notify_amount):
             available_amount = amount_response.body.data.available_amount
             amount = float(available_amount.replace(',',''))
             consul_kv.put_kv(f'ConsulManager/exp/lists/alicloud/{account}/amount',{'amount':amount})
-            print('alicloud',account,f'可用余额:{available_amount}', flush=True)
+            #print('alicloud',account,f'可用余额:{available_amount}', flush=True)
             amount_dict = {}
             if amount < notify_amount:
                 amount_dict = {'amount':amount}
@@ -92,7 +92,7 @@ def exp(account,collect_days,notify_days,notify_amount):
             title = '阿里云资源到期通知'
             md = content
             notify.feishu(feishuwh,title,md)
-    print(f"=====【阿里云：余额与到期日统计结束：{account}】", flush=True)
+    #print(f"=====【阿里云：余额与到期日统计结束：{account}】", flush=True)
 
 def group(account):
     ak,sk = consul_kv.get_aksk('alicloud',account)

flask-consul/units/consul_kv.py

@@ -80,6 +80,17 @@ def get_services_meta(services_name):
     else:
         return {'code': 50000, 'data': f'{response.status_code}:{response.text}'}
 
+def get_services_list_by_region(services_name,region):
+    region = f'and "{region}" in Tags'
+    url = f'{consul_url}/agent/services?filter=Service == "{services_name}" {region}'
+    response = requests.get(url, headers=headers)
+    if response.status_code == 200:
+        info = response.json()
+        res_list = info.keys()
+        return res_list
+    else:
+        return []
+
 def get_aksk(cloud,account):
     import myaes
     aksk_dict = get_value(f'ConsulManager/assets/{cloud}/aksk/{account}')

flask-consul/units/gen_config.py

@@ -108,7 +108,7 @@ groups:
     annotations:
       description: "{{ $labels.group }}_{{ $labels.name }}：每分钟慢查询:{{ $value }} \\n> {{ $labels.iid }}"
 
-  - alert: mysql_当前活跃的连接数过多
+  - alert: MySQL_当前活跃的连接数过多
     expr: mysql_global_status_threads_running > 100
     for: 1m
     labels:
@@ -116,7 +116,7 @@ groups:
     annotations:
       description: "{{ $labels.group }}_{{ $labels.name }}：当前活跃的连接数:{{ $value }} \\n> {{ $labels.iid }}"
 
-  - alert: mysql_当前updating状态的线程过多
+  - alert: MySQL_当前updating状态的线程过多
     expr: mysql_info_schema_processlist_threads{state=~"updating"} > 100
     for: 1m
     labels:
@@ -140,6 +140,30 @@ groups:
     annotations:
       description: "{{ $labels.group }}_{{ $labels.name }}：Mysql Connections:{{ $value | humanize }} \\n> {{ $labels.iid }}"
 
+  - alert: MySQL_主从IO线程运行状态异常
+  expr: mysql_slave_status_master_server_id > 0 and ON (instance) mysql_slave_status_slave_io_running == 0
+  for: 1m
+  labels:
+    severity: critical
+  annotations:
+    description: "{{ $labels.group }}_{{ $labels.name }}：MySQL Slave IO thread not running \\n> {{ $labels.iid }}"
+  
+  - alert: MySQL_主从SQL线程运行状态异常
+  expr: mysql_slave_status_master_server_id > 0 and ON (instance) mysql_slave_status_slave_sql_running == 0
+  for: 1m
+  labels:
+    severity: critical
+  annotations:
+    description: "{{ $labels.group }}_{{ $labels.name }}：MySQL Slave SQL thread not running \\n> {{ $labels.iid }}"
+
+  - alert: MySQL_主从复制延迟过高
+  expr: mysql_slave_status_seconds_behind_master > 3
+  for: 1m
+  labels:
+    severity: critical
+  annotations:
+    description: "{{ $labels.group }}_{{ $labels.name }}：主从复制延迟当前:{{ $value | humanize }}s \\n> {{ $labels.iid }}"
+
   - alert: MySQL_is_Restart
     expr: mysql_global_status_uptime <600
     for: 2m

flask-consul/units/prom/mysql_huawei.py

@@ -0,0 +1,48 @@
+from huaweicloudsdkcore.auth.credentials import BasicCredentials
+from huaweicloudsdkces.v1.region.ces_region import CesRegion
+from huaweicloudsdkcore.exceptions import exceptions
+from huaweicloudsdkces.v1 import *
+from datetime import datetime
+from units import consul_kv
+
+def exporter(vendor,account,region):
+    ak,sk = consul_kv.get_aksk(vendor,account)
+    credentials = BasicCredentials(ak, sk)
+    client = CesClient.new_builder() \
+        .with_credentials(credentials) \
+        .with_region(CesRegion.value_of(region)) \
+        .build()
+    metric_name_dict = {"rds001_cpu_util":["# HELP mysql_cpu_util CPU使用率","# TYPE mysql_cpu_util gauge"],
+                        "rds002_mem_util":["# HELP mysql_mem_util 内存使用率","# TYPE mysql_mem_util gauge"],
+                        "rds003_iops":["# HELP mysql_iops_count 每秒I/O请求数","# TYPE mysql_iops_count gauge"],
+                        "rds039_disk_util":["# HELP mysql_disk_util 磁盘使用率","# TYPE mysql_disk_util gauge"],
+                        "rds081_vm_ioutils":["# HELP mysql_io_util 磁盘I/O使用率","# TYPE mysql_io_util gauge"]
+                       }
+    metric_body_list = []
+    now = int(datetime.now().timestamp()*1000)
+    rds_list = consul_kv.get_services_list_by_region(f'{vendor}_{account}_rds',region)
+    try:
+        for i in metric_name_dict.keys():
+            for rdsid in rds_list:
+                metric_body_list.append(MetricInfo(namespace="SYS.RDS",metric_name=i,dimensions=[MetricsDimension(name="rds_cluster_id",value=rdsid)]))
+
+        request = BatchListMetricDataRequest()
+        request.body = BatchListMetricDataRequestBody(to=now,_from=now-600000,filter="max",period="1",metrics=metric_body_list)
+        response = client.batch_list_metric_data(request).to_dict()
+        for i in response['metrics']:
+            rdsid= i['dimensions'][0]['value']
+            value = i['datapoints'][-1]['max']
+            ts = i['datapoints'][-1]['timestamp']
+            metric = i['metric_name']
+            prom_metric_name = metric_name_dict[metric][0].split()[2]
+            metric_name_dict[metric].append(f'{prom_metric_name}{{iid="{rdsid}"}} {float(value)} {ts}')
+        prom_metric_list = []
+        for x in metric_name_dict.values():
+            prom_metric_list = prom_metric_list + x
+        return prom_metric_list
+    except exceptions.ClientRequestException as e:
+        print(e.status_code,flush=True)
+        print(e.request_id,flush=True)
+        print(e.error_code,flush=True)
+        print(e.error_msg,flush=True)
+

flask-consul/views/prom/cloud_mysql_metrics.py

@@ -0,0 +1,14 @@
+from flask import Blueprint,Response
+from flask_restful import reqparse, Resource, Api
+from config import vendors,regions
+from units import token_auth,consul_kv
+from units.prom import mysql_huawei
+import json
+blueprint = Blueprint('cloud_mysql_metrics',__name__)
+api = Api(blueprint)
+
+class Exporter(Resource):
+    def get(self,vendor,account,region):
+        prom_metric_list = mysql_huawei.exporter(vendor,account,region)
+        return Response('\n'.join(prom_metric_list).encode('utf-8'),mimetype="text/plain")
+api.add_resource(Exporter, '/api/cloud_mysql_metrics/<vendor>/<account>/<region>')