增加Exporter功能，可以直接从云厂商获取DRS的CPU、内存、磁盘指标
StarsL.cn
parent
8b10f170b5
commit
fae5169340
|
|
@ -20,8 +20,10 @@ alibabacloud_resourcemanager20200331==2.1.1
|
|||
alibabacloud_ecs20140526==2.1.1
|
||||
alibabacloud_rds20140815==2.1.1
|
||||
alibabacloud_bssopenapi20171214==2.0.5
|
||||
tencentcloud-sdk-python-common==3.0.763
|
||||
tencentcloud-sdk-python-cvm==3.0.763
|
||||
tencentcloud-sdk-python-cdb==3.0.763
|
||||
tencentcloud-sdk-python-dcdb==3.0.763
|
||||
tencentcloud-sdk-python-billing==3.0.763
|
||||
aliyun-python-sdk-cms==7.0.32
|
||||
tencentcloud-sdk-python-common==3.0.770
|
||||
tencentcloud-sdk-python-cvm==3.0.770
|
||||
tencentcloud-sdk-python-cdb==3.0.770
|
||||
tencentcloud-sdk-python-dcdb==3.0.770
|
||||
tencentcloud-sdk-python-billing==3.0.770
|
||||
tencentcloud-sdk-python-monitor==3.0.770
|
||||
|
|
|
|||
|
|
@ -1,7 +1,23 @@
|
|||
from config import consul_token,consul_url
|
||||
|
||||
def rds_config(services_list, exporter):
|
||||
def rds_config(region_list,cm_exporter,services_list,exporter):
|
||||
region_str = '\n - '.join([i.replace('/rds','') for i in region_list])
|
||||
consul_server = consul_url.split("/")[2]
|
||||
exporter_config = f"""
|
||||
- job_name: 'ConsulManager-exporter'
|
||||
scrape_interval: 30s
|
||||
scrape_timeout: 15s
|
||||
static_configs:
|
||||
- targets:
|
||||
- {region_str}
|
||||
relabel_configs:
|
||||
- source_labels: [__address__]
|
||||
target_label: __metrics_path__
|
||||
regex: (.*)
|
||||
replacement: /api/cloud_mysql_metrics/${{1}}
|
||||
- target_label: __address__
|
||||
replacement: {cm_exporter}
|
||||
"""
|
||||
configs = f"""
|
||||
- job_name: multi_mysqld_exporter
|
||||
scrape_interval: 15s
|
||||
|
|
@ -44,7 +60,17 @@ def rds_config(services_list, exporter):
|
|||
- source_labels: ['__meta_consul_service_metadata_itype']
|
||||
target_label: itype
|
||||
"""
|
||||
return {'code': 20000,'configs': configs }
|
||||
if not services_list:
|
||||
return {'code': 20000,'configs': '请选择需要Prometheus从Conusl自动发现的MySQL组' }
|
||||
if services_list and exporter == '':
|
||||
return {'code': 20000,'configs': '您已经选择了需要Prometheus从Conusl自动发现MySQL组，\n请输入Mysql_Exporter的地址和端口，例如：10.0.0.26:9104' }
|
||||
if region_list and cm_exporter == '':
|
||||
return {'code': 20000,'configs': '您已经选择了需要从云监控采集基础指标(CPU、内存、磁盘、IO)的MySQL组，\n请输入ConsulManager地址和端口，例如：10.0.0.26:1026' }
|
||||
|
||||
if region_list:
|
||||
return {'code': 20000,'configs': exporter_config + configs }
|
||||
else:
|
||||
return {'code': 20000,'configs': configs }
|
||||
|
||||
def ecs_config(services_list,ostype_list):
|
||||
consul_server = consul_url.split("/")[2]
|
||||
|
|
|
|||
|
|
@ -0,0 +1,34 @@
|
|||
from aliyunsdkcore.client import AcsClient
|
||||
from aliyunsdkcore.acs_exception.exceptions import ClientException
|
||||
from aliyunsdkcore.acs_exception.exceptions import ServerException
|
||||
from aliyunsdkcms.request.v20190101.DescribeMetricLastRequest import DescribeMetricLastRequest
|
||||
from datetime import datetime
|
||||
from units import consul_kv
|
||||
import json
|
||||
|
||||
def exporter(vendor,account,region):
|
||||
ak,sk = consul_kv.get_aksk(vendor,account)
|
||||
client_rdsmonit = AcsClient(ak, sk, region)
|
||||
request_rdsmonit = DescribeMetricLastRequest()
|
||||
request_rdsmonit.set_accept_format('json')
|
||||
request_rdsmonit.set_Namespace("acs_rds_dashboard")
|
||||
metric_name_dict = {"CpuUsage":["# HELP mysql_cpu_util CPU使用率","# TYPE mysql_cpu_util gauge"],
|
||||
"MemoryUsage":["# HELP mysql_mem_util 内存使用率","# TYPE mysql_mem_util gauge"],
|
||||
"DiskUsage":["# HELP mysql_disk_util 磁盘使用率","# TYPE mysql_disk_util gauge"],
|
||||
"IOPSUsage":["# HELP mysql_io_util 磁盘I/O使用率","# TYPE mysql_io_util gauge"]
|
||||
}
|
||||
try:
|
||||
for i in metric_name_dict.keys():
|
||||
request_rdsmonit.set_MetricName(i)
|
||||
response_rdsmonit = json.loads(client_rdsmonit.do_action_with_exception(request_rdsmonit), encoding='utf-8')
|
||||
instance = json.loads(response_rdsmonit["Datapoints"])
|
||||
prom_metric_name = metric_name_dict[i][0].split()[2]
|
||||
for j in instance:
|
||||
iid,max,ts = j["instanceId"],j["Maximum"],j["timestamp"]
|
||||
metric_name_dict[i].append(f'{prom_metric_name}{{iid="{iid}"}} {float(max)} {ts}')
|
||||
prom_metric_list = []
|
||||
for x in metric_name_dict.values():
|
||||
prom_metric_list = prom_metric_list + x
|
||||
return prom_metric_list
|
||||
except Exception as e:
|
||||
print(e,flush=True)
|
||||
|
|
@ -0,0 +1,44 @@
|
|||
import json
|
||||
from tencentcloud.common import credential
|
||||
from tencentcloud.common.profile.client_profile import ClientProfile
|
||||
from tencentcloud.common.profile.http_profile import HttpProfile
|
||||
from tencentcloud.common.exception.tencent_cloud_sdk_exception import TencentCloudSDKException
|
||||
from tencentcloud.monitor.v20180724 import monitor_client, models
|
||||
from datetime import datetime,timedelta
|
||||
from units import consul_kv
|
||||
|
||||
def exporter(vendor,account,region):
|
||||
ak,sk = consul_kv.get_aksk(vendor,account)
|
||||
cred = credential.Credential(ak,sk)
|
||||
client = monitor_client.MonitorClient(cred, region)
|
||||
req = models.GetMonitorDataRequest()
|
||||
metric_name_dict = {"CpuUseRate":["# HELP mysql_cpu_util CPU使用率","# TYPE mysql_cpu_util gauge"],
|
||||
"MemoryUseRate":["# HELP mysql_mem_util 内存使用率","# TYPE mysql_mem_util gauge"],
|
||||
"IOPS":["# HELP mysql_iops_count 每秒I/O请求数","# TYPE mysql_iops_count gauge"],
|
||||
"VolumeRate":["# HELP mysql_disk_util 磁盘使用率","# TYPE mysql_disk_util gauge"],
|
||||
"IopsUseRate":["# HELP mysql_io_util 磁盘I/O使用率","# TYPE mysql_io_util gauge"]
|
||||
}
|
||||
rds_list = consul_kv.get_services_list_by_region(f'{vendor}_{account}_rds',region)
|
||||
rds_list = list(rds_list)
|
||||
rds_list_10 = [rds_list[i:i + 10] for i in range(0, len(rds_list), 10)]
|
||||
try:
|
||||
for i in metric_name_dict.keys():
|
||||
for rdss in rds_list_10:
|
||||
starttime = (datetime.now() + timedelta(minutes=-1)).strftime('%Y-%m-%dT%H:%M:%S+08:00')
|
||||
ins_list = [{"Dimensions":[{"Name":"InstanceId","Value":x}]} for x in rdss]
|
||||
params = {"Namespace":"QCE/CDB","MetricName":i,"Period":60,"StartTime":starttime,"Instances":ins_list}
|
||||
req.from_json_string(json.dumps(params))
|
||||
resp = client.GetMonitorData(req)
|
||||
metric_list = resp.DataPoints
|
||||
for metrics in metric_list:
|
||||
iid = metrics.Dimensions[0].Value
|
||||
value = metrics.Values[-1]
|
||||
ts = metrics.Timestamps[-1]*1000
|
||||
prom_metric_name = metric_name_dict[i][0].split()[2]
|
||||
metric_name_dict[i].append(f'{prom_metric_name}{{iid="{iid}"}} {float(value)} {ts}')
|
||||
prom_metric_list = []
|
||||
for x in metric_name_dict.values():
|
||||
prom_metric_list = prom_metric_list + x
|
||||
return prom_metric_list
|
||||
except TencentCloudSDKException as err:
|
||||
print(err)
|
||||
|
|
@ -89,7 +89,7 @@ class Nodes(Resource):
|
|||
elif stype == 'rdspconfig':
|
||||
args = parser.parse_args()
|
||||
services_dict = args['services_dict']
|
||||
return gen_config.rds_config(services_dict['services_list'],services_dict['exporter'])
|
||||
return gen_config.rds_config(services_dict['jobrds_list'],services_dict['cm_exporter'],services_dict['services_list'],services_dict['exporter'])
|
||||
elif stype == 'cstecs':
|
||||
args = parser.parse_args()
|
||||
cst_ecs_dict = args['cst_ecs_dict']
|
||||
|
|
|
|||
|
|
@ -2,13 +2,18 @@ from flask import Blueprint,Response
|
|||
from flask_restful import reqparse, Resource, Api
|
||||
from config import vendors,regions
|
||||
from units import token_auth,consul_kv
|
||||
from units.prom import mysql_huawei
|
||||
from units.prom import mysql_huawei,mysql_ali,mysql_tencent
|
||||
import json
|
||||
blueprint = Blueprint('cloud_mysql_metrics',__name__)
|
||||
api = Api(blueprint)
|
||||
|
||||
class Exporter(Resource):
|
||||
def get(self,vendor,account,region):
|
||||
prom_metric_list = mysql_huawei.exporter(vendor,account,region)
|
||||
if vendor == 'huaweicloud':
|
||||
prom_metric_list = mysql_huawei.exporter(vendor,account,region)
|
||||
elif vendor == 'alicloud':
|
||||
prom_metric_list = mysql_ali.exporter(vendor,account,region)
|
||||
elif vendor == 'tencent_cloud':
|
||||
prom_metric_list = mysql_tencent.exporter(vendor,account,region)
|
||||
return Response('\n'.join(prom_metric_list).encode('utf-8'),mimetype="text/plain")
|
||||
api.add_resource(Exporter, '/api/cloud_mysql_metrics/<vendor>/<account>/<region>')
|
||||
|
|
|
|||
|
|
@ -46,7 +46,7 @@ Object.keys(filters).forEach(key => {
|
|||
})
|
||||
|
||||
Vue.config.productionTip = false
|
||||
Vue.prototype.VER = 'v0.9.4'
|
||||
Vue.prototype.VER = 'v0.9.5'
|
||||
|
||||
new Vue({
|
||||
el: '#app',
|
||||
|
|
|
|||
|
|
@ -1,10 +1,13 @@
|
|||
<template>
|
||||
<div class="app-container">
|
||||
<el-select v-model="services" multiple placeholder="请选择需要生成配置的服务" filterable collapse-tags clearable style="width: 350px" class="filter-item">
|
||||
<el-select v-model="services" multiple placeholder="选择需要自动发现的MySQL组" filterable collapse-tags clearable style="width: 260px" class="filter-item">
|
||||
<el-option v-for="item in services_list" :key="item" :label="item" :value="item" />
|
||||
</el-select>
|
||||
<font color="#ff0000">*</font>MySQLd_Exporter IP端口
|
||||
<el-input v-model="exporter" placeholder="x.x.x.x:9104" clearable style="width: 200px;" class="filter-item" />
|
||||
<el-input v-model="exporter" placeholder="Mysqld_Exporter IP端口" clearable style="width: 200px;" class="filter-item" />
|
||||
<el-select v-model="jobrds" multiple placeholder="选择需要采集指标的MySQL组" filterable collapse-tags clearable style="width: 260px" class="filter-item">
|
||||
<el-option v-for="item in jobrds_list" :key="item" :label="item" :value="item" />
|
||||
</el-select>
|
||||
<el-input v-model="cm_exporter" placeholder="ConsulManager IP端口" clearable style="width: 190px;" class="filter-item" />
|
||||
<el-button class="filter-item" type="primary" icon="el-icon-magic-stick" @click="fetchRdsConfig">
|
||||
生成配置
|
||||
</el-button>
|
||||
|
|
@ -16,17 +19,20 @@
|
|||
</template>
|
||||
|
||||
<script>
|
||||
import { getRdsServicesList, getRdsConfig } from '@/api/node-exporter'
|
||||
import { getRdsServicesList, getRdsConfig, getJobRds } from '@/api/node-exporter'
|
||||
export default {
|
||||
data() {
|
||||
return {
|
||||
listLoading: false,
|
||||
services: [],
|
||||
jobrds: [],
|
||||
ostype: [],
|
||||
services_list: [],
|
||||
services_dict: {},
|
||||
jobrds_list: [],
|
||||
exporter: '',
|
||||
configs: ''
|
||||
cm_exporter: '',
|
||||
configs: '该功能用于生成Prometheus的两个JOB配置，生成后请复制到Prometheus配置中：\n\n1. 选择需要同步的账号，Prometheus即可自动发现该账号下的所有DRS实例。\n\n2. 由于Mysqld_Exporter无法监控到云数据库的CPU、内存、磁盘的使用情况，所以ConsulManager开发了Exporter功能，可以直接从云厂商获取到这些指标并接入Prometheus！\n 选择需要采集指标的RDS账号区域，即可生成Prometheus的JOB配置。'
|
||||
}
|
||||
},
|
||||
created() {
|
||||
|
|
@ -46,13 +52,18 @@ export default {
|
|||
this.listLoading = true
|
||||
getRdsServicesList().then(response => {
|
||||
this.services_list = response.services_list
|
||||
this.listLoading = false
|
||||
})
|
||||
getJobRds().then(response => {
|
||||
this.jobrds_list = response.jobrds
|
||||
})
|
||||
this.listLoading = false
|
||||
},
|
||||
fetchRdsConfig() {
|
||||
this.listLoading = true
|
||||
this.services_dict.services_list = this.services
|
||||
this.services_dict.exporter = this.exporter
|
||||
this.services_dict.jobrds_list = this.jobrds
|
||||
this.services_dict.cm_exporter = this.cm_exporter
|
||||
getRdsConfig(this.services_dict).then(response => {
|
||||
this.configs = response.configs
|
||||
this.listLoading = false
|
||||
|
|
|
|||
Loading…
Reference in New Issue