diff --git a/flask-consul/manager.py b/flask-consul/manager.py index 032149c..bd5d341 100755 --- a/flask-consul/manager.py +++ b/flask-consul/manager.py @@ -8,6 +8,7 @@ if consul_kv.get_kv_dict(skey_path) == {}: consul_kv.put_kv(skey_path,{'sk':''.join(str(uuid.uuid4()).split('-'))}) from views import login, blackbox, consul, jobs, nodes, selfnode, avd, exp, jms, edit_cloud +from views.prom import cloud_mysql_metrics from units.cloud import huaweicloud,alicloud,tencent_cloud from units.avd import avd_list from units.jms import sync_jms @@ -26,6 +27,7 @@ app.register_blueprint(avd.blueprint) app.register_blueprint(exp.blueprint) app.register_blueprint(jms.blueprint) app.register_blueprint(edit_cloud.blueprint) +app.register_blueprint(cloud_mysql_metrics.blueprint) class Config(object): JOBS = [] diff --git a/flask-consul/requirements.txt b/flask-consul/requirements.txt index 652a57c..acc7a3e 100644 --- a/flask-consul/requirements.txt +++ b/flask-consul/requirements.txt @@ -10,11 +10,12 @@ xlrd==1.2.0 #pyDes==2.0.1 pycryptodome==3.14.1 beautifulsoup4==4.11.1 -huaweicloudsdkcore==3.1.5 -huaweicloudsdkecs==3.1.5 -huaweicloudsdkeps==3.1.5 -huaweicloudsdkbss==3.1.5 -huaweicloudsdkrds==3.1.5 +huaweicloudsdkcore==3.1.8 +huaweicloudsdkecs==3.1.8 +huaweicloudsdkeps==3.1.8 +huaweicloudsdkbss==3.1.8 +huaweicloudsdkrds==3.1.8 +huaweicloudsdkces==3.1.8 alibabacloud_resourcemanager20200331==2.1.1 alibabacloud_ecs20140526==2.1.1 alibabacloud_rds20140815==2.1.1 diff --git a/flask-consul/units/cloud/alicloud.py b/flask-consul/units/cloud/alicloud.py index b54cf75..6a00340 100644 --- a/flask-consul/units/cloud/alicloud.py +++ b/flask-consul/units/cloud/alicloud.py @@ -19,7 +19,7 @@ from units.cloud import sync_rds from units.cloud import notify def exp(account,collect_days,notify_days,notify_amount): - print(f"=====【阿里云：余额与到期日统计开始：{account}】", flush=True) + #print(f"=====【阿里云：余额与到期日统计开始：{account}】", flush=True) ak,sk = consul_kv.get_aksk('alicloud',account) now = datetime.datetime.utcnow().strftime('%Y-%m-%dT16:00:00Z') collect = (datetime.datetime.utcnow() + datetime.timedelta(days=collect_days+1)).strftime('%Y-%m-%dT16:00:00Z') @@ -36,7 +36,7 @@ def exp(account,collect_days,notify_days,notify_amount): available_amount = amount_response.body.data.available_amount amount = float(available_amount.replace(',','')) consul_kv.put_kv(f'ConsulManager/exp/lists/alicloud/{account}/amount',{'amount':amount}) - print('alicloud',account,f'可用余额:{available_amount}', flush=True) + #print('alicloud',account,f'可用余额:{available_amount}', flush=True) amount_dict = {} if amount < notify_amount: amount_dict = {'amount':amount} @@ -92,7 +92,7 @@ def exp(account,collect_days,notify_days,notify_amount): title = '阿里云资源到期通知' md = content notify.feishu(feishuwh,title,md) - print(f"=====【阿里云：余额与到期日统计结束：{account}】", flush=True) + #print(f"=====【阿里云：余额与到期日统计结束：{account}】", flush=True) def group(account): ak,sk = consul_kv.get_aksk('alicloud',account) diff --git a/flask-consul/units/consul_kv.py b/flask-consul/units/consul_kv.py index 2441b69..f7661cc 100644 --- a/flask-consul/units/consul_kv.py +++ b/flask-consul/units/consul_kv.py @@ -80,6 +80,17 @@ def get_services_meta(services_name): else: return {'code': 50000, 'data': f'{response.status_code}:{response.text}'} +def get_services_list_by_region(services_name,region): + region = f'and "{region}" in Tags' + url = f'{consul_url}/agent/services?filter=Service == "{services_name}" {region}' + response = requests.get(url, headers=headers) + if response.status_code == 200: + info = response.json() + res_list = info.keys() + return res_list + else: + return [] + def get_aksk(cloud,account): import myaes aksk_dict = get_value(f'ConsulManager/assets/{cloud}/aksk/{account}') diff --git a/flask-consul/units/gen_config.py b/flask-consul/units/gen_config.py index 8db3993..d3fc1f2 100644 --- a/flask-consul/units/gen_config.py +++ b/flask-consul/units/gen_config.py @@ -108,7 +108,7 @@ groups: annotations: description: "{{ $labels.group }}_{{ $labels.name }}：每分钟慢查询:{{ $value }} \\n> {{ $labels.iid }}" - - alert: mysql_当前活跃的连接数过多 + - alert: MySQL_当前活跃的连接数过多 expr: mysql_global_status_threads_running > 100 for: 1m labels: @@ -116,7 +116,7 @@ groups: annotations: description: "{{ $labels.group }}_{{ $labels.name }}：当前活跃的连接数:{{ $value }} \\n> {{ $labels.iid }}" - - alert: mysql_当前updating状态的线程过多 + - alert: MySQL_当前updating状态的线程过多 expr: mysql_info_schema_processlist_threads{state=~"updating"} > 100 for: 1m labels: @@ -140,6 +140,30 @@ groups: annotations: description: "{{ $labels.group }}_{{ $labels.name }}：Mysql Connections:{{ $value | humanize }} \\n> {{ $labels.iid }}" + - alert: MySQL_主从IO线程运行状态异常 + expr: mysql_slave_status_master_server_id > 0 and ON (instance) mysql_slave_status_slave_io_running == 0 + for: 1m + labels: + severity: critical + annotations: + description: "{{ $labels.group }}_{{ $labels.name }}：MySQL Slave IO thread not running \\n> {{ $labels.iid }}" + + - alert: MySQL_主从SQL线程运行状态异常 + expr: mysql_slave_status_master_server_id > 0 and ON (instance) mysql_slave_status_slave_sql_running == 0 + for: 1m + labels: + severity: critical + annotations: + description: "{{ $labels.group }}_{{ $labels.name }}：MySQL Slave SQL thread not running \\n> {{ $labels.iid }}" + + - alert: MySQL_主从复制延迟过高 + expr: mysql_slave_status_seconds_behind_master > 3 + for: 1m + labels: + severity: critical + annotations: + description: "{{ $labels.group }}_{{ $labels.name }}：主从复制延迟当前:{{ $value | humanize }}s \\n> {{ $labels.iid }}" + - alert: MySQL_is_Restart expr: mysql_global_status_uptime <600 for: 2m diff --git a/flask-consul/units/prom/mysql_huawei.py b/flask-consul/units/prom/mysql_huawei.py new file mode 100644 index 0000000..ae6fa95 --- /dev/null +++ b/flask-consul/units/prom/mysql_huawei.py @@ -0,0 +1,48 @@ +from huaweicloudsdkcore.auth.credentials import BasicCredentials +from huaweicloudsdkces.v1.region.ces_region import CesRegion +from huaweicloudsdkcore.exceptions import exceptions +from huaweicloudsdkces.v1 import * +from datetime import datetime +from units import consul_kv + +def exporter(vendor,account,region): + ak,sk = consul_kv.get_aksk(vendor,account) + credentials = BasicCredentials(ak, sk) + client = CesClient.new_builder() \ + .with_credentials(credentials) \ + .with_region(CesRegion.value_of(region)) \ + .build() + metric_name_dict = {"rds001_cpu_util":["# HELP mysql_cpu_util CPU使用率","# TYPE mysql_cpu_util gauge"], + "rds002_mem_util":["# HELP mysql_mem_util 内存使用率","# TYPE mysql_mem_util gauge"], + "rds003_iops":["# HELP mysql_iops_count 每秒I/O请求数","# TYPE mysql_iops_count gauge"], + "rds039_disk_util":["# HELP mysql_disk_util 磁盘使用率","# TYPE mysql_disk_util gauge"], + "rds081_vm_ioutils":["# HELP mysql_io_util 磁盘I/O使用率","# TYPE mysql_io_util gauge"] + } + metric_body_list = [] + now = int(datetime.now().timestamp()*1000) + rds_list = consul_kv.get_services_list_by_region(f'{vendor}_{account}_rds',region) + try: + for i in metric_name_dict.keys(): + for rdsid in rds_list: + metric_body_list.append(MetricInfo(namespace="SYS.RDS",metric_name=i,dimensions=[MetricsDimension(name="rds_cluster_id",value=rdsid)])) + + request = BatchListMetricDataRequest() + request.body = BatchListMetricDataRequestBody(to=now,_from=now-600000,filter="max",period="1",metrics=metric_body_list) + response = client.batch_list_metric_data(request).to_dict() + for i in response['metrics']: + rdsid= i['dimensions'][0]['value'] + value = i['datapoints'][-1]['max'] + ts = i['datapoints'][-1]['timestamp'] + metric = i['metric_name'] + prom_metric_name = metric_name_dict[metric][0].split()[2] + metric_name_dict[metric].append(f'{prom_metric_name}{{iid="{rdsid}"}} {float(value)} {ts}') + prom_metric_list = [] + for x in metric_name_dict.values(): + prom_metric_list = prom_metric_list + x + return prom_metric_list + except exceptions.ClientRequestException as e: + print(e.status_code,flush=True) + print(e.request_id,flush=True) + print(e.error_code,flush=True) + print(e.error_msg,flush=True) + diff --git a/flask-consul/views/prom/cloud_mysql_metrics.py b/flask-consul/views/prom/cloud_mysql_metrics.py new file mode 100644 index 0000000..e15115e --- /dev/null +++ b/flask-consul/views/prom/cloud_mysql_metrics.py @@ -0,0 +1,14 @@ +from flask import Blueprint,Response +from flask_restful import reqparse, Resource, Api +from config import vendors,regions +from units import token_auth,consul_kv +from units.prom import mysql_huawei +import json +blueprint = Blueprint('cloud_mysql_metrics',__name__) +api = Api(blueprint) + +class Exporter(Resource): + def get(self,vendor,account,region): + prom_metric_list = mysql_huawei.exporter(vendor,account,region) + return Response('\n'.join(prom_metric_list).encode('utf-8'),mimetype="text/plain") +api.add_resource(Exporter, '/api/cloud_mysql_metrics///')