github
/
ConsulManager
mirror of https://github.com/starsliao/ConsulManager
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

add alicloud redis and redis metrics

pull/46/head
starsliao 2022-11-24 10:24:58 +08:00
parent 148ebdca88
commit 40c4e86364
13 changed files with 150 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
flask-consul/requirements.txt
View File

@ -19,9 +19,10 @@ huaweicloudsdkrds==3.1.11
huaweicloudsdkces==3.1.11
huaweicloudsdkdcs==3.1.11
alibabacloud_resourcemanager20200331==2.1.1
alibabacloud_ecs20140526==2.1.1
alibabacloud_rds20140815==2.1.1
alibabacloud_bssopenapi20171214==2.0.5
alibabacloud_ecs20140526==2.1.3
alibabacloud_rds20140815==2.1.2
alibabacloud_r_kvstore20150101==2.20.7
alibabacloud_bssopenapi20171214==2.0.6
aliyun-python-sdk-cms==7.0.32
tencentcloud-sdk-python-common==3.0.770
tencentcloud-sdk-python-cvm==3.0.770

56
flask-consul/units/cloud/alicloud.py
View File

@ -11,12 +11,12 @@ from alibabacloud_bssopenapi20171214.client import Client as BssOpenApi20171214C
from alibabacloud_bssopenapi20171214 import models as bss_open_api_20171214_models
from alibabacloud_rds20140815.client import Client as Rds20140815Client
from alibabacloud_rds20140815 import models as rds_20140815_models
from alibabacloud_r_kvstore20150101 import models as r_kvstore_20150101_models
from alibabacloud_r_kvstore20150101.client import Client as R_kvstore20150101Client
import sys,datetime,hashlib
from units import consul_kv,consul_svc
from units.cloud import sync_ecs
from units.cloud import sync_rds
from units.cloud import notify
from units.cloud import sync_ecs,sync_rds,sync_redis,notify
def exp(account,collect_days,notify_days,notify_amount):
#print(f"=====【阿里云：余额与到期日统计开始：{account}】", flush=True)
@ -178,6 +178,56 @@ def ecs(account,region,isextip=False):
data = {'count':'无','update':f'失败','status':50000,'msg':str(e)}
consul_kv.put_kv(f'ConsulManager/record/jobs/alicloud/{account}/ecs/{region}', data)
def redis(account,region):
ak,sk = consul_kv.get_aksk('alicloud',account)
now = datetime.datetime.now().strftime('%m.%d/%H:%M')
group_dict = consul_kv.get_value(f'ConsulManager/assets/alicloud/group/{account}')
config = open_api_models.Config(access_key_id=ak,access_key_secret=sk)
config.endpoint = 'r-kvstore.aliyuncs.com'
client = R_kvstore20150101Client(config)
try:
runtime = util_models.RuntimeOptions()
describe_instances_request = r_kvstore_20150101_models.DescribeInstancesRequest(
page_size=100,
region_id=region
)
redisbaseinfo = client.describe_instances_with_options(describe_instances_request, runtime)
redisbase_list = redisbaseinfo.body.to_map()['Instances']["KVStoreInstance"]
redis_dict = {i['InstanceId']:{'name':i.get('InstanceName',f"未命名{i['InstanceId']}"),
'domain':i['ConnectionDomain'],
'ip':i['PrivateIp'],
'port':i['Port'],
'region':region,
'group':group_dict.get(i['ResourceGroupId'],'无'),
'status':i['InstanceStatus'],
'itype':i['ArchitectureType'],
'ver':i['EngineVersion'],
'mem':f"{i['Capacity']}MB",
'exp': '-' if i['EndTime'] == None else i['EndTime'].split('T')[0]
} for i in redisbase_list}
count = len(redis_dict)
off,on = sync_redis.w2consul('alicloud',account,region,redis_dict)
data = {'count':count,'update':now,'status':20000,'on':on,'off':off,'msg':f'redis同步成功！总数：{count}，开机：{on}，关机：{off}'}
consul_kv.put_kv(f'ConsulManager/record/jobs/alicloud/{account}/redis/{region}', data)
print('【JOB】===>', 'alicloud_redis', account,region, data, flush=True)
except TeaException as e:
emsg = e.message.split('. ',1)[0]
print("【code:】",e.code,"\n【message:】",emsg, flush=True)
data = consul_kv.get_value(f'ConsulManager/record/jobs/alicloud/{account}/redis/{region}')
if data == {}:
data = {'count':'无','update':f'失败{e.code}','status':50000,'msg':emsg}
else:
data['update'] = f'失败{e.code}'
data['msg'] = emsg
consul_kv.put_kv(f'ConsulManager/record/jobs/alicloud/{account}/redis/{region}', data)
except Exception as e:
data = {'count':'无','update':f'失败','status':50000,'msg':str(e)}
consul_kv.put_kv(f'ConsulManager/record/jobs/alicloud/{account}/redis/{region}', data)
def rds(account,region):
ak,sk = consul_kv.get_aksk('alicloud',account)
now = datetime.datetime.now().strftime('%m.%d/%H:%M')

2
flask-consul/units/cloud/sync_redis.py
View File

@ -25,7 +25,7 @@ def w2consul(vendor,account,region,redis_dict):
for k,v in redis_dict.items():
iid = k
#对consul中关机的redis做标记。
if v['status'] in ['SHUTDOWN']:
if v['status'] in ['SHUTDOWN','Unavailable','Inactive','Released']:
off = off + 1
tags = ['OFF', v['itype'], v['ver'], region]
stat = 'off'

3
flask-consul/units/prom/mysql_ali.py
View File

@ -15,7 +15,8 @@ def exporter(vendor,account,region):
metric_name_dict = {"CpuUsage":["# HELP mysql_cpu_util CPU使用率","# TYPE mysql_cpu_util gauge"],
"MemoryUsage":["# HELP mysql_mem_util 内存使用率","# TYPE mysql_mem_util gauge"],
"DiskUsage":["# HELP mysql_disk_util 磁盘使用率","# TYPE mysql_disk_util gauge"],
"IOPSUsage":["# HELP mysql_io_util 磁盘I/O使用率","# TYPE mysql_io_util gauge"]
"IOPSUsage":["# HELP mysql_io_util 磁盘I/O使用率","# TYPE mysql_io_util gauge"],
"ConnectionUsage":["# HELP mysql_conn_util 连接数使用率","# TYPE mysql_conn_util gauge"]
}
for i in metric_name_dict.keys():
request_rdsmonit.set_MetricName(i)

4
flask-consul/units/prom/mysql_huawei.py
View File

@ -14,10 +14,10 @@ def exporter(vendor,account,region):
.build()
metric_name_dict = {"rds001_cpu_util":["# HELP mysql_cpu_util CPU使用率","# TYPE mysql_cpu_util gauge"],
"rds002_mem_util":["# HELP mysql_mem_util 内存使用率","# TYPE mysql_mem_util gauge"],
"rds003_iops":["# HELP mysql_iops_count 每秒I/O请求数","# TYPE mysql_iops_count gauge"],
"rds039_disk_util":["# HELP mysql_disk_util 磁盘使用率","# TYPE mysql_disk_util gauge"],
"rds074_slow_queries":["# HELP mysql_slow_queries 每分钟慢SQL","# TYPE mysql_slow_queries gauge"],
"rds081_vm_ioutils":["# HELP mysql_io_util 磁盘I/O使用率","# TYPE mysql_io_util gauge"]
"rds081_vm_ioutils":["# HELP mysql_io_util 磁盘I/O使用率","# TYPE mysql_io_util gauge"],
"rds072_conn_usage":["# HELP mysql_conn_util 连接数使用率","# TYPE mysql_conn_util gauge"]
}
metric_body_list = []
now = int(datetime.now().timestamp()*1000)

4
flask-consul/units/prom/mysql_tencent.py
View File

@ -14,9 +14,9 @@ def exporter(vendor,account,region):
req = models.GetMonitorDataRequest()
metric_name_dict = {"CpuUseRate":["# HELP mysql_cpu_util CPU使用率","# TYPE mysql_cpu_util gauge"],
"MemoryUseRate":["# HELP mysql_mem_util 内存使用率","# TYPE mysql_mem_util gauge"],
"IOPS":["# HELP mysql_iops_count 每秒I/O请求数","# TYPE mysql_iops_count gauge"],
"VolumeRate":["# HELP mysql_disk_util 磁盘使用率","# TYPE mysql_disk_util gauge"],
"IopsUseRate":["# HELP mysql_io_util 磁盘I/O使用率","# TYPE mysql_io_util gauge"]
"IopsUseRate":["# HELP mysql_io_util 磁盘I/O使用率","# TYPE mysql_io_util gauge"],
"ConnectionUseRate":["# HELP mysql_conn_util 连接数使用率","# TYPE mysql_conn_util gauge"]
}
rds_list = consul_kv.get_services_list_by_region(f'{vendor}_{account}_rds',region)
rds_list = list(rds_list)

32
flask-consul/units/prom/redis_ali.py Normal file
View File

@ -0,0 +1,32 @@
from aliyunsdkcore.client import AcsClient
from aliyunsdkcore.acs_exception.exceptions import ClientException
from aliyunsdkcore.acs_exception.exceptions import ServerException
from aliyunsdkcms.request.v20190101.DescribeMetricLastRequest import DescribeMetricLastRequest
from datetime import datetime
from units import consul_kv
import json
def exporter(vendor,account,region):
ak,sk = consul_kv.get_aksk(vendor,account)
client_redismonit = AcsClient(ak, sk, region)
request_redismonit = DescribeMetricLastRequest()
request_redismonit.set_accept_format('json')
request_redismonit.set_Namespace("acs_kvstore")
metric_name_dict = {"CpuUsage":["# HELP redis_cpu_util CPU使用率","# TYPE redis_cpu_util gauge"],
"MemoryUsage":["# HELP redis_mem_util 内存使用率","# TYPE redis_mem_util gauge"],
"ConnectionUsage":["# HELP redis_conn_util 连接数使用率","# TYPE redis_conn_util gauge"],
"IntranetInRatio":["# HELP redis_netin_util 写入带宽使用率","# TYPE redis_netin_util gauge"],
"IntranetOutRatio":["# HELP redis_netout_util 读取带宽使用率","# TYPE redis_netout_util gauge"]
}
for i in metric_name_dict.keys():
request_redismonit.set_MetricName(i)
response_redismonit = json.loads(client_redismonit.do_action_with_exception(request_redismonit))
instance = json.loads(response_redismonit["Datapoints"])
prom_metric_name = metric_name_dict[i][0].split()[2]
for j in instance:
iid,max,ts = j["instanceId"],j["Maximum"],j["timestamp"]
metric_name_dict[i].append(f'{prom_metric_name}{{iid="{iid}"}} {float(max)} {ts}')
prom_metric_list = []
for x in metric_name_dict.values():
prom_metric_list = prom_metric_list + x
return prom_metric_list

4
flask-consul/units/prom/redis_huawei.py
View File

@ -17,7 +17,7 @@ def exporter(vendor,account,region):
"keyspace_hits_perc":["# HELP redis_hits_util 缓存命中率","# TYPE redis_hits_util gauge"],
"total_connections_received":["# HELP redis_newconn_count 每分钟新建的连接数","# TYPE redis_newconn_count gauge"],
"rx_controlled":["# HELP redis_rx_controlled 每分钟被流控的次数","# TYPE redis_rx_controlled gauge"],
"is_slow_log_exist":["# HELP redis_slow_log 慢日志情况","# TYPE redis_slow_log gauge"]
"is_slow_log_exist":["# HELP redis_slow_log 慢日志情况","# TYPE redis_slow_log gauge"],
}
metric_body_list = []
now = int(datetime.now().timestamp()*1000)
@ -28,7 +28,7 @@ def exporter(vendor,account,region):
metric_body_list.append(MetricInfo(namespace="SYS.DCS",metric_name=i,dimensions=[MetricsDimension(name="dcs_instance_id",value=id)]))
request = BatchListMetricDataRequest()
request.body = BatchListMetricDataRequestBody(to=now,_from=now-120000,filter="max",period="1",metrics=metric_body_list)
request.body = BatchListMetricDataRequestBody(to=now,_from=now-180000,filter="max",period="1",metrics=metric_body_list)
#print(now-300000,now)
response = client.batch_list_metric_data(request).to_dict()
for i in response['metrics']:

46
flask-consul/units/prom/redis_tencent.py Normal file
View File

@ -0,0 +1,46 @@
import json
from tencentcloud.common import credential
from tencentcloud.common.profile.client_profile import ClientProfile
from tencentcloud.common.profile.http_profile import HttpProfile
from tencentcloud.common.exception.tencent_cloud_sdk_exception import TencentCloudSDKException
from tencentcloud.monitor.v20180724 import monitor_client, models
from datetime import datetime,timedelta
from units import consul_kv
def exporter(vendor,account,region):
ak,sk = consul_kv.get_aksk(vendor,account)
cred = credential.Credential(ak,sk)
client = monitor_client.MonitorClient(cred, region)
req = models.GetMonitorDataRequest()
metric_name_dict = {"CpuMaxUtil":["# HELP redis_cpu_util 实例中节点最大CPU使用率","# TYPE redis_cpu_util gauge"],
"MemMaxUtil":["# HELP redis_mem_util 实例中节点最大内存使用率","# TYPE redis_mem_util gauge"],
"ConnectionsUtil":["# HELP redis_conn_util 连接使用率","# TYPE redis_conn_util gauge"],
"CmdBigValue":["# HELP redis_big_count 每秒请求命令大小超过32KB的执行次数","# TYPE redis_big_count gauge"],
"CmdSlow":["# HELP redis_slow_count 执行时延大于slowlog-log-slower-than配置的命令次数","# TYPE redis_slow_count gauge"],
"InFlowLimit":["# HELP redis_inlimit_count 入流量触发限流的次数","# TYPE redis_inlimit_count gauge"],
"OutFlowLimit":["# HELP redis_outlimit_count 出流量触发限流的次数","# TYPE redis_outlimit_count gauge"]
}
redis_list = consul_kv.get_services_list_by_region(f'{vendor}_{account}_redis',region)
redis_list = list(redis_list)
redis_list_10 = [redis_list[i:i + 10] for i in range(0, len(redis_list), 10)]
try:
for i in metric_name_dict.keys():
for rediss in redis_list_10:
starttime = (datetime.now() + timedelta(minutes=-1)).strftime('%Y-%m-%dT%H:%M:%S+08:00')
ins_list = [{"Dimensions":[{"Name":"InstanceId","Value":x}]} for x in rediss]
params = {"Namespace":"QCE/REDIS_MEM","MetricName":i,"Period":60,"StartTime":starttime,"Instances":ins_list}
req.from_json_string(json.dumps(params))
resp = client.GetMonitorData(req)
metric_list = resp.DataPoints
for metrics in metric_list:
iid = metrics.Dimensions[0].Value
value = metrics.Values[-1]
ts = metrics.Timestamps[-1]*1000
prom_metric_name = metric_name_dict[i][0].split()[2]
metric_name_dict[i].append(f'{prom_metric_name}{{iid="{iid}"}} {float(value)} {ts}')
prom_metric_list = []
for x in metric_name_dict.values():
prom_metric_list = prom_metric_list + x
return prom_metric_list
except TencentCloudSDKException as err:
print(err)

6
flask-consul/views/prom/cloud_metrics.py
View File

@ -2,7 +2,7 @@ from flask import Blueprint,Response
from flask_restful import reqparse, Resource, Api
from config import vendors,regions
from units import token_auth,consul_kv
from units.prom import mysql_huawei,mysql_ali,mysql_tencent,redis_huawei
from units.prom import mysql_huawei,mysql_ali,mysql_tencent,redis_huawei,redis_ali
import json
blueprint = Blueprint('cloud_metrics',__name__)
api = Api(blueprint)
@ -21,8 +21,8 @@ class RedisExporter(Resource):
def get(self,vendor,account,region):
if vendor == 'huaweicloud':
prom_metric_list = redis_huawei.exporter(vendor,account,region)
#elif vendor == 'alicloud':
#prom_metric_list = mysql_ali.exporter(vendor,account,region)
elif vendor == 'alicloud':
prom_metric_list = redis_ali.exporter(vendor,account,region)
#elif vendor == 'tencent_cloud':
#prom_metric_list = mysql_tencent.exporter(vendor,account,region)
return Response('\n'.join(prom_metric_list).encode('utf-8'),mimetype="text/plain")

2
vue-consul/src/views/rds/pconfig.vue
View File

@ -32,7 +32,7 @@ export default {
jobrds_list: [],
exporter: '',
cm_exporter: '',
configs: '该功能用于生成Prometheus的两个JOB配置，生成后请复制到Prometheus配置中：\n\n1. 选择需要同步的账号，Prometheus即可自动发现该账号下的所有DRS实例。\n\n2. 由于Mysqld_Exporter无法监控到云数据库的CPU、内存、磁盘的使用情况，所以ConsulManager开发了Exporter功能，配置到Prometheus即可直接从云厂商采集到这些指标！\n 选择需要采集指标的RDS账号区域，即可生成Prometheus的JOB配置。'
configs: '该功能用于生成Prometheus的两个JOB配置，生成后请复制到Prometheus配置中：\n\n1. 选择需要同步的账号，Prometheus即可自动发现该账号下的所有MySQL实例。\n\n2. 由于Mysqld_Exporter无法监控到云数据库的CPU、内存、磁盘的使用情况，所以ConsulManager开发了Exporter功能，配置到Prometheus即可直接从云厂商采集到这些指标！\n 选择需要采集指标的RDS账号区域，即可生成Prometheus的JOB配置。'
}
},
created() {

2
vue-consul/src/views/redis/lists.vue
View File

@ -59,7 +59,7 @@
</el-table-column>
<el-table-column prop="ver" label="版本" sortable align="center" width="80" />
<el-table-column prop="mem" label="内存" sortable align="center" width="90" />
<el-table-column prop="exp" label="到期日" sortable align="center" width="90" />
<el-table-column prop="exp" label="到期日" sortable align="center" width="95" />
<el-table-column prop="itype" label="类型" sortable align="center" width="120" show-overflow-tooltip />
<el-table-column prop="iid" label="实例ID" sortable align="center" width="150" show-overflow-tooltip />
<el-table-column label="操作" align="center" width="120" class-name="small-padding fixed-width">

2
vue-consul/src/views/redis/pconfig.vue
View File

@ -32,7 +32,7 @@ export default {
jobredis_list: [],
exporter: '',
cm_exporter: '',
configs: '该功能用于生成Prometheus的两个JOB配置，生成后请复制到Prometheus配置中：\n\n1. 选择需要同步的账号，Prometheus即可自动发现该账号下的所有DRS实例。\n\n2. 由于Redis_Exporter无法监控到云数据库的CPU、内存、磁盘的使用情况，所以ConsulManager开发了Exporter功能，配置到Prometheus即可直接从云厂商采集到这些指标！\n 选择需要采集指标的REDIS账号区域，即可生成Prometheus的JOB配置。'
configs: '该功能用于生成Prometheus的两个JOB配置，生成后请复制到Prometheus配置中：\n\n1. 选择需要同步的账号，Prometheus即可自动发现该账号下的所有REDIS实例。\n\n2. 由于Redis_Exporter无法监控到云数据库的CPU、内存、磁盘的使用情况，所以ConsulManager开发了Exporter功能，配置到Prometheus即可直接从云厂商采集到这些指标！\n 选择需要采集指标的REDIS账号区域，即可生成Prometheus的JOB配置。'
}
},
created() {