From 1e4398b84561687afad7fd6229c107797eb757d4 Mon Sep 17 00:00:00 2001 From: "StarsL.cn" Date: Sun, 8 May 2022 20:37:24 +0800 Subject: [PATCH] add selfnode --- blackbox_exporter/README.md | 10 - docs/ECS主机监控.md | 10 + FAQ.md => docs/FAQ.md | 0 docs/blackbox站点监控.md | 4 +- flask-consul/config.py | 3 +- flask-consul/manager.py | 4 +- flask-consul/units/gen_config.py | 2 + flask-consul/units/selfnode_manager.py | 69 +++ flask-consul/views/selfnode.py | 54 ++ units/{input.py => blackbox-input.py} | 6 +- units/blackbox-instance.list | 5 + units/instance.list | 1 - units/selfnode-input.py | 31 + units/selfnode-instance.list | 4 + vue-consul/.dockerignore | 1 + vue-consul/src/api/selfnode.js | 37 ++ vue-consul/src/router/index.js | 12 +- vue-consul/src/views/blackbox/index.vue | 12 +- vue-consul/src/views/dashboard/index.vue | 9 + vue-consul/src/views/login/index.vue | 2 +- vue-consul/src/views/node-exporter/jobs.vue | 6 +- .../src/views/node-exporter/pconfig.vue | 1 + vue-consul/src/views/node-exporter/self.vue | 533 ++++++++++++++++++ 23 files changed, 791 insertions(+), 25 deletions(-) delete mode 100644 blackbox_exporter/README.md rename FAQ.md => docs/FAQ.md (100%) create mode 100644 flask-consul/units/selfnode_manager.py create mode 100644 flask-consul/views/selfnode.py rename units/{input.py => blackbox-input.py} (84%) create mode 100644 units/blackbox-instance.list delete mode 100644 units/instance.list create mode 100755 units/selfnode-input.py create mode 100644 units/selfnode-instance.list create mode 100644 vue-consul/src/api/selfnode.js create mode 100644 vue-consul/src/views/node-exporter/self.vue diff --git a/blackbox_exporter/README.md b/blackbox_exporter/README.md deleted file mode 100644 index 292954b..0000000 --- a/blackbox_exporter/README.md +++ /dev/null @@ -1,10 +0,0 @@ -### Blackbox Exporter Dashboard -- 支持Grafana 8，基于blackbox_exporter 0.19.0设计 -- 采用图表+曲线图方式展示TCP，ICMP，HTTPS的服务状态，各阶段请求延时，HTTPS证书信息等 -- 优化展示效果，支持监控目标的分组、分类级联展示，多服务同时对比展示。 - -导入ID：9965 -详细URL：https://grafana.com/grafana/dashboards/9965 - - -![1](https://raw.githubusercontent.com/starsliao/ConsulManager/main/screenshot/1.png)![2](https://raw.githubusercontent.com/starsliao/ConsulManager/main/screenshot/2.png) diff --git a/docs/ECS主机监控.md b/docs/ECS主机监控.md index 80b72a9..183d209 100644 --- a/docs/ECS主机监控.md +++ b/docs/ECS主机监控.md @@ -26,6 +26,16 @@ - **导入ID：8919** - 详细URL：[https://grafana.com/grafana/dashboards/8919](https://grafana.com/grafana/dashboards/8919) +--- + +### 批量导入自建主机脚本 + +在units目录下`selfnode-instance.list`中写入监控目标的信息：机房/公司 租户/部门 区域/项目 分组/环境 名称 实例(ip:端口) 系统(linux/windows)，每行一个，空格分隔。 + +**注意：前5个字段组合起来必须唯一，作为一个监控项的ID。即Consul的ServiceID** + +修改units目录下导入脚本中的consul_token和consul_url，保存后执行selfnode-input.py，即可导入所有监控目标到Consul，并符合Prometheus的自动发现配置。 + ### 注意： ##### 主动关机的ECS，会在同步时候从Consul中清除，即会在Prometheus中去除(减少无效的告警)，重新开机后会增加回去。 diff --git a/FAQ.md b/docs/FAQ.md similarity index 100% rename from FAQ.md rename to docs/FAQ.md diff --git a/docs/blackbox站点监控.md b/docs/blackbox站点监控.md index b57d8c4..d1fbf0f 100644 --- a/docs/blackbox站点监控.md +++ b/docs/blackbox站点监控.md @@ -34,8 +34,8 @@ #### 批量导入脚本 -在units目录下`instance.list`中写入监控目标的信息：JOB名称，公司/部门，项目，环境，名称，实例url，每行一个，空格分隔。 +在units目录下`blackbox-instance.list`中写入监控目标的信息：监控类型，公司/部门，项目，环境，名称，实例url，每行一个，空格分隔。 **注意：前5个字段组合起来必须唯一，作为一个监控项的ID。即Consul的ServiceID** -修改units目录下导入脚本中的consul_token和consul_url，保存后执行input.py，即可导入所有监控目标到Consul，并符合Prometheus的自动发现配置。 +修改units目录下导入脚本中的consul_token和consul_url，保存后执行blackbox-input.py，即可导入所有监控目标到Consul，并符合Prometheus的自动发现配置。 diff --git a/flask-consul/config.py b/flask-consul/config.py index 6aaa00c..08623c1 100644 --- a/flask-consul/config.py +++ b/flask-consul/config.py @@ -13,7 +13,8 @@ regions = {'huaweicloud':{'none': '无','cn-east-3': '华东-上海一','cn-east 'cn-wulanchabu':'华北6(乌兰察布)', 'cn-hangzhou':'华东1(杭州)', 'cn-shanghai':'华东2(上海)', 'cn-shenzhen':'华南1(深圳)', 'cn-heyuan':'华南2(河源)', 'cn-guangzhou':'华南3(广州)', 'cn-chengdu':'西南1(成都)', 'cn-hongkong':'中国(香港)', - 'cn-nanjing':'华东5(南京-本地地域)', 'us-east-1':'美国东部1(弗吉尼亚)'}, + 'cn-nanjing':'华东5(南京-本地地域)', 'us-east-1':'美国东部1(弗吉尼亚)','us-west-1':'美国(硅谷)', + 'eu-west-1':'英国(伦敦)','ap-southeast-1':'新加坡','ap-northeast-1':'日本(东京)'}, 'tencent_cloud':{'none': '无',"ap-nanjing":"华东地区(南京)","ap-shanghai":"华东地区(上海)", "ap-guangzhou":"华南地区(广州)","ap-beijing":"华北地区(北京)","ap-tianjin":"华北地区(天津)", "ap-chengdu":"西南地区(成都)","ap-chongqing":"西南地区(重庆)", diff --git a/flask-consul/manager.py b/flask-consul/manager.py index 607a224..b0e6f09 100755 --- a/flask-consul/manager.py +++ b/flask-consul/manager.py @@ -6,7 +6,7 @@ skey_path = 'ConsulManager/assets/secret/skey' if consul_kv.get_kv_dict(skey_path) == {}: consul_kv.put_kv(skey_path,{'sk':''.join(str(uuid.uuid4()).split('-'))}) -from views import login, blackbox, consul, jobs, nodes +from views import login, blackbox, consul, jobs, nodes, selfnode from units.cloud import huaweicloud,alicloud,tencent_cloud app = Flask(__name__) app.register_blueprint(login.blueprint) @@ -14,7 +14,7 @@ app.register_blueprint(blackbox.blueprint) app.register_blueprint(consul.blueprint) app.register_blueprint(jobs.blueprint) app.register_blueprint(nodes.blueprint) - +app.register_blueprint(selfnode.blueprint) class Config(object): JOBS = [] SCHEDULER_API_ENABLED = True diff --git a/flask-consul/units/gen_config.py b/flask-consul/units/gen_config.py index 9d63e8c..0117089 100644 --- a/flask-consul/units/gen_config.py +++ b/flask-consul/units/gen_config.py @@ -16,6 +16,8 @@ def ecs_config(services_list,ostype_list): services: {services_list} tags: ['{ostype}'] relabel_configs: + - source_labels: ['__meta_consul_service'] + target_label: cservice - source_labels: ['__meta_consul_service_metadata_vendor'] target_label: vendor - source_labels: ['__meta_consul_service_metadata_region'] diff --git a/flask-consul/units/selfnode_manager.py b/flask-consul/units/selfnode_manager.py new file mode 100644 index 0000000..75865a0 --- /dev/null +++ b/flask-consul/units/selfnode_manager.py @@ -0,0 +1,69 @@ +import requests,json +import sys +sys.path.append("..") +from config import consul_token,consul_url + +headers = {'X-Consul-Token': consul_token} + +def get_all_list(vendor,account,region,group): + vendor = f'and Meta.vendor=="{vendor}"' if vendor != '' else f'and Meta.vendor != ""' + account = f'and Meta.account=="{account}"' if account != '' else f'and Meta.account != ""' + region = f'and Meta.region=="{region}"' if region != '' else f'and Meta.region != ""' + group = f'and Meta.group=="{group}"' if group != '' else f'and Meta.group != ""' + url = f'{consul_url}/agent/services?filter=Service == selfnode_exporter {vendor} {account} {region} {group}' + response = requests.get(url, headers=headers) + if response.status_code == 200: + info = response.json() + all_list = [i['Meta'] for i in info.values()] + vendor_list = sorted(list(set([i['vendor'] for i in all_list]))) + account_list = sorted(list(set([i['account'] for i in all_list]))) + region_list = sorted(list(set([i['region'] for i in all_list]))) + group_list = sorted(list(set([i['group'] for i in all_list]))) + return {'code': 20000,'all_list':all_list,'vendor_list':vendor_list, + 'account_list':account_list,'region_list':region_list,'group_list':group_list} + else: + return {'code': 50000, 'data': f'{response.status_code}:{response.text}'} + +def get_service(): + response = requests.get(f'{consul_url}/agent/services?filter=Service == selfnode_exporter', headers=headers) + if response.status_code == 200: + info = response.json() + all_list = [i['Meta'] for i in info.values()] + vendor_list = sorted(list(set([i['vendor'] for i in all_list]))) + account_list = sorted(list(set([i['account'] for i in all_list]))) + region_list = sorted(list(set([i['region'] for i in all_list]))) + group_list = sorted(list(set([i['group'] for i in all_list]))) + return {'code': 20000,'all_list':all_list,'vendor_list':vendor_list, + 'account_list':account_list,'region_list':region_list,'group_list':group_list} + else: + return {'code': 50000, 'data': f'{response.status_code}:{response.text}'} + +def add_service(vendor,account,region,group,name,ip,port,os): + sid = f"{vendor}/{account}/{region}/{group}@{name}" + instance = f'{ip}:{port}' + if '//' in sid or sid.startswith('/') or sid.endswith('/'): + return {"code": 50000, "data": f"服务ID【{sid}】首尾不能包含'/'，并且不能包含两个连续的'/'"} + data = { + "id": sid, + "name": 'selfnode_exporter', + 'Address': ip, + 'port': int(port), + "tags": [vendor,os], + "Meta": {'vendor':vendor,'account':account,'region':region,'group':group, + 'name':name,'instance':instance,'os':os}, + "check": {"tcp": instance,"interval": "60s"} + } + reg = requests.put(f'{consul_url}/agent/service/register', headers=headers, data=json.dumps(data)) + if reg.status_code == 200: + return {"code": 20000, "data": f"【{sid}】增加成功！"} + else: + return {"code": 50000, "data": f"{reg.status_code}【{sid}】{reg.text}"} + +def del_service(vendor,account,region,group,name): + sid = f"{vendor}/{account}/{region}/{group}@{name}" + reg = requests.put(f'{consul_url}/agent/service/deregister/{sid}', headers=headers) + if reg.status_code == 200: + return {"code": 20000, "data": f"【{sid}】删除成功！"} + else: + return {"code": 50000, "data": f"{reg.status_code}【{sid}】{reg.text}"} + diff --git a/flask-consul/views/selfnode.py b/flask-consul/views/selfnode.py new file mode 100644 index 0000000..80e86f7 --- /dev/null +++ b/flask-consul/views/selfnode.py @@ -0,0 +1,54 @@ +from flask import Blueprint +from flask_restful import reqparse, Resource, Api +import sys +sys.path.append("..") +from units import token_auth,selfnode_manager + +blueprint = Blueprint('selfnode',__name__) +api = Api(blueprint) + +parser = reqparse.RequestParser() +parser.add_argument('vendor',type=str) +parser.add_argument('account',type=str) +parser.add_argument('region',type=str) +parser.add_argument('group',type=str) +parser.add_argument('name',type=str) +parser.add_argument('ip',type=str) +parser.add_argument('port',type=str) +parser.add_argument('os',type=str) +parser.add_argument('del_dict',type=dict) +parser.add_argument('up_dict',type=dict) + +class GetAllList(Resource): + @token_auth.auth.login_required + def get(self): + args = parser.parse_args() + return selfnode_manager.get_all_list(args['vendor'],args['account'],args['region'],args['group']) + +class SelfnodeApi(Resource): + decorators = [token_auth.auth.login_required] + def get(self): + return selfnode_manager.get_service() + def post(self): + args = parser.parse_args() + return selfnode_manager.add_service(args['vendor'],args['account'],args['region'], + args['group'],args['name'],args['ip'],args['port'],args['os']) + def put(self): + args = parser.parse_args() + del_dict = args['del_dict'] + up_dict = args['up_dict'] + resp_del = selfnode_manager.del_service(del_dict['vendor'],del_dict['account'], + del_dict['region'],del_dict['group'],del_dict['name']) + resp_add = selfnode_manager.add_service(up_dict['vendor'],up_dict['account'],up_dict['region'], + up_dict['group'],up_dict['name'],up_dict['ip'], + up_dict['port'],up_dict['os']) + if resp_del["code"] == 20000 and resp_add["code"] == 20000: + return {"code": 20000, "data": f"更新成功！"} + else: + return {"code": 50000, "data": f"更新失败！"} + def delete(self): + args = parser.parse_args() + return selfnode_manager.del_service(args['vendor'],args['account'],args['region'],args['group'],args['name']) + +api.add_resource(GetAllList,'/api/selfnode/alllist') +api.add_resource(SelfnodeApi, '/api/selfnode/service') diff --git a/units/input.py b/units/blackbox-input.py similarity index 84% rename from units/input.py rename to units/blackbox-input.py index 61bf11c..f043d9a 100755 --- a/units/input.py +++ b/units/blackbox-input.py @@ -1,11 +1,13 @@ #!/usr/bin/python3 import requests,json -consul_token = 'xxxxxxxxxx' +consul_token = 'xxxxxxxxxx' #Consul SecretID consul_url = 'http://x.x.x.x:8500/v1' -with open('instance.list', 'r') as file: +with open('blackbox-instance.list', 'r') as file: lines = file.readlines() for line in lines: + if line.startswith('#'): + continue module,company,project,env,name,instance = line.split() headers = {'X-Consul-Token': consul_token} data = { diff --git a/units/blackbox-instance.list b/units/blackbox-instance.list new file mode 100644 index 0000000..585b590 --- /dev/null +++ b/units/blackbox-instance.list @@ -0,0 +1,5 @@ +# 监控类型和Blackbox配置中的module名必须保持一致 +# 每行需要有6个字段，空格分隔： +# 监控类型 公司/部门 项目 环境 名称 实例url +# 以下为例子，#开头的行不会被导入，需要导入的行不要以#开头： +# http_2xx 阿里巴巴 电商 prod 淘宝 https://www.taobao.com diff --git a/units/instance.list b/units/instance.list deleted file mode 100644 index d1489a8..0000000 --- a/units/instance.list +++ /dev/null @@ -1 +0,0 @@ -JOB名称 公司/部门 项目 环境 名称 实例url diff --git a/units/selfnode-input.py b/units/selfnode-input.py new file mode 100755 index 0000000..ecfe37b --- /dev/null +++ b/units/selfnode-input.py @@ -0,0 +1,31 @@ +#!/usr/bin/python3 +import requests,json +consul_token = 'xxxxxxxxxx' #Consul SecretID +consul_url = 'http://x.x.x.x:8500/v1' + +with open('selfnode-instance.list', 'r') as file: + lines = file.readlines() + for line in lines: + if line.startswith('#'): + continue + vendor,account,region,group,name,instance,os = line.split() + headers = {'X-Consul-Token': consul_token} + sid = f"{vendor}/{account}/{region}/{group}@{name}" + ip = instance.split(':')[0] + port = instance.split(':')[1] + data = { + "id": sid, + "name": 'selfnode_exporter', + 'Address': ip, + 'port': int(port), + "tags": [vendor,os], + "Meta": {'vendor':vendor,'account':account,'region':region,'group':group, + 'name':name,'instance':instance,'os':os}, + "check": {"tcp": instance,"interval": "60s"} + } + + reg = requests.put(f"{consul_url}/agent/service/register", headers=headers, data=json.dumps(data)) + if reg.status_code == 200: + print({"code": 20000,"data": "增加成功！"}) + else: + print({"code": 50000,"data": f'{reg.status_code}:{reg.text}'}) diff --git a/units/selfnode-instance.list b/units/selfnode-instance.list new file mode 100644 index 0000000..eb4234d --- /dev/null +++ b/units/selfnode-instance.list @@ -0,0 +1,4 @@ +# 每行需要有7个字段，空格分隔： +# 机房/公司 租户/部门 区域/项目 分组/环境 名称 实例(ip:端口) 系统(linux/windows) +# 以下为例子，#开头的行不会被导入，需要导入的行不要以#开头： +# openstack 运维 深圳 prod 监控主机 10.0.0.26:9100 linux diff --git a/vue-consul/.dockerignore b/vue-consul/.dockerignore index 3c3629e..d5f19d8 100644 --- a/vue-consul/.dockerignore +++ b/vue-consul/.dockerignore @@ -1 +1,2 @@ node_modules +package-lock.json diff --git a/vue-consul/src/api/selfnode.js b/vue-consul/src/api/selfnode.js new file mode 100644 index 0000000..688a0b6 --- /dev/null +++ b/vue-consul/src/api/selfnode.js @@ -0,0 +1,37 @@ +import request from '@/utils/request-ops' + +export function getAllList(vendor, account, region, group) { + return request({ + url: '/api/selfnode/alllist', + method: 'get', + params: { vendor, account, region, group } + }) +} + +export function getAllInfo() { + return request({ + url: '/api/selfnode/service', + method: 'get' + }) +} +export function addService(data) { + return request({ + url: '/api/selfnode/service', + method: 'post', + data + }) +} +export function updateService(del_dict, up_dict) { + return request({ + url: '/api/selfnode/service', + method: 'put', + data: { del_dict, up_dict } + }) +} +export function delService(data) { + return request({ + url: '/api/selfnode/service', + method: 'delete', + data + }) +} diff --git a/vue-consul/src/router/index.js b/vue-consul/src/router/index.js index 4b4c640..9095255 100644 --- a/vue-consul/src/router/index.js +++ b/vue-consul/src/router/index.js @@ -86,8 +86,8 @@ export const constantRoutes = [ path: '/nodes', component: Layout, redirect: '/nodes/jobs', - name: 'ECS 云主机监控', - meta: { title: 'ECS 云主机监控', icon: 'example' }, + name: 'Node 主机监控', + meta: { title: 'Node 主机监控', icon: 'example' }, children: [ { path: 'jobs', @@ -99,7 +99,13 @@ export const constantRoutes = [ path: 'lists', name: '云主机列表', component: () => import('@/views/node-exporter/lists'), - meta: { title: '云主机列表', icon: 'el-icon-s-platform' } + meta: { title: '云主机列表', icon: 'el-icon-cloudy' } + }, + { + path: 'self', + name: '自建主机管理', + component: () => import('@/views/node-exporter/self'), + meta: { title: '自建主机管理', icon: 'el-icon-s-platform' } }, { path: 'pconfig', diff --git a/vue-consul/src/views/blackbox/index.vue b/vue-consul/src/views/blackbox/index.vue index a888dda..22f97f1 100644 --- a/vue-consul/src/views/blackbox/index.vue +++ b/vue-consul/src/views/blackbox/index.vue @@ -94,7 +94,15 @@ - + + + + 监控类型 + + + + + @@ -113,7 +121,7 @@ 实例 - + diff --git a/vue-consul/src/views/dashboard/index.vue b/vue-consul/src/views/dashboard/index.vue index ec076d6..1552ac2 100644 --- a/vue-consul/src/views/dashboard/index.vue +++ b/vue-consul/src/views/dashboard/index.vue @@ -4,6 +4,15 @@ StarsL.cn + + +

v0.5.1

+

Node 主机监控可以方便在页面上管理自建主机了。

+

优化了导入脚本，并且支持了自建主机的批量导入。

+

云主机同步增加了部分国外的区域。

+

web页面的描述做了优化，修复了一些bug。

+ +

v0.5.0

diff --git a/vue-consul/src/views/login/index.vue b/vue-consul/src/views/login/index.vue index d4fc4ee..a1a5bf9 100644 --- a/vue-consul/src/views/login/index.vue +++ b/vue-consul/src/views/login/index.vue @@ -46,7 +46,7 @@
- v0.5.0 + v0.5.1
diff --git a/vue-consul/src/views/node-exporter/jobs.vue b/vue-consul/src/views/node-exporter/jobs.vue index 7ba4fac..565e8dd 100644 --- a/vue-consul/src/views/node-exporter/jobs.vue +++ b/vue-consul/src/views/node-exporter/jobs.vue @@ -223,7 +223,11 @@ export default { { value: 'cn-chengdu', label: '西南1(成都)' }, { value: 'cn-hongkong', label: '中国(香港)' }, { value: 'cn-nanjing', label: '华东5(南京-本地地域)' }, - { value: 'us-east-1', label: '美国东部1(弗吉尼亚)' } + { value: 'us-east-1', label: '美国东部1(弗吉尼亚)' }, + { value: 'us-west-1', label: '美国(硅谷)' }, + { value: 'eu-west-1', label: '英国(伦敦)' }, + { value: 'ap-southeast-1', label: '新加坡' }, + { value: 'ap-northeast-1', label: '日本(东京)' } ], tencent_cloud: [ { value: 'ap-nanjing', label: '华东地区(南京)' }, diff --git a/vue-consul/src/views/node-exporter/pconfig.vue b/vue-consul/src/views/node-exporter/pconfig.vue index fe7584e..93547eb 100644 --- a/vue-consul/src/views/node-exporter/pconfig.vue +++ b/vue-consul/src/views/node-exporter/pconfig.vue @@ -47,6 +47,7 @@ export default { this.listLoading = true getServicesList().then(response => { this.services_list = response.services_list + this.services_list.push('selfnode_exporter') this.listLoading = false }) }, diff --git a/vue-consul/src/views/node-exporter/self.vue b/vue-consul/src/views/node-exporter/self.vue new file mode 100644 index 0000000..feb2a20 --- /dev/null +++ b/vue-consul/src/views/node-exporter/self.vue @@ -0,0 +1,533 @@ + + +