218 lines
8.2 KiB
Python
218 lines
8.2 KiB
Python
import requests,json,consul_kv
|
||
from config import consul_token,consul_url
|
||
|
||
headers = {'X-Consul-Token': consul_token}
|
||
init_module_list = ['http_2xx','http_4xx','tcp_connect','icmp','http200igssl','httpNoRedirect4ssl','http_5xx','http_post_2xx','ssh_banner']
|
||
def get_all_list(module,company,project,env):
|
||
module = f'and Meta.module=="{module}"' if module != '' else f'and Meta.module != ""'
|
||
company = f'and Meta.company=="{company}"' if company != '' else f'and Meta.company != ""'
|
||
project = f'and Meta.project=="{project}"' if project != '' else f'and Meta.project != ""'
|
||
env = f'and Meta.env=="{env}"' if env != '' else f'and Meta.env != ""'
|
||
url = f'{consul_url}/agent/services?filter=Service == blackbox_exporter {module} {company} {project} {env}'
|
||
response = requests.get(url, headers=headers)
|
||
if response.status_code == 200:
|
||
info = response.json()
|
||
all_list = [i['Meta'] for i in info.values()]
|
||
module_list = consul_kv.get_value('ConsulManager/record/blackbox/module_list')['module_list']
|
||
company_list = sorted(list(set([i['company'] for i in all_list])))
|
||
project_list = sorted(list(set([i['project'] for i in all_list])))
|
||
env_list = sorted(list(set([i['env'] for i in all_list])))
|
||
|
||
init_m_list = [x for x in init_module_list if x not in module_list]
|
||
module_list = module_list + ['---'] + init_m_list
|
||
|
||
return {'code': 20000,'all_list':all_list,'module_list':module_list,
|
||
'company_list':company_list,'project_list':project_list,'env_list':env_list}
|
||
else:
|
||
return {'code': 50000, 'data': f'{response.status_code}:{response.text}'}
|
||
|
||
def get_service():
|
||
response = requests.get(f'{consul_url}/agent/services?filter=Service == blackbox_exporter', headers=headers)
|
||
if response.status_code == 200:
|
||
info = response.json()
|
||
all_list = [i['Meta'] for i in info.values()]
|
||
module_list = sorted(list(set([i['module'] for i in all_list])))
|
||
company_list = sorted(list(set([i['company'] for i in all_list])))
|
||
project_list = sorted(list(set([i['project'] for i in all_list])))
|
||
env_list = sorted(list(set([i['env'] for i in all_list])))
|
||
consul_kv.put_kv('ConsulManager/record/blackbox/module_list',{'module_list':module_list})
|
||
init_m_list = [x for x in init_module_list if x not in module_list]
|
||
module_list = module_list + ['------'] + init_m_list
|
||
|
||
return {'code': 20000,'all_list':all_list,'module_list':module_list,
|
||
'company_list':company_list,'project_list':project_list,'env_list':env_list}
|
||
else:
|
||
return {'code': 50000, 'data': f'{response.status_code}:{response.text}'}
|
||
|
||
def add_service(module,company,project,env,name,instance):
|
||
sid = f"{module}/{company}/{project}/{env}@{name}"
|
||
if '//' in sid or sid.startswith('/') or sid.endswith('/'):
|
||
return {"code": 50000, "data": f"服务ID【{sid}】首尾不能包含'/'，并且不能包含两个连续的'/'"}
|
||
data = {
|
||
"id": sid,
|
||
"name": 'blackbox_exporter',
|
||
"tags": [module],
|
||
"Meta": {'module':module,'company':company,'project':project,'env':env,'name':name,'instance':instance}
|
||
}
|
||
reg = requests.put(f'{consul_url}/agent/service/register', headers=headers, data=json.dumps(data))
|
||
if reg.status_code == 200:
|
||
return {"code": 20000, "data": f"【{sid}】增加成功！"}
|
||
else:
|
||
return {"code": 50000, "data": f"{reg.status_code}【{sid}】{reg.text}"}
|
||
|
||
def del_service(module,company,project,env,name):
|
||
sid = f"{module}/{company}/{project}/{env}@{name}"
|
||
reg = requests.put(f'{consul_url}/agent/service/deregister/{sid}', headers=headers)
|
||
if reg.status_code == 200:
|
||
return {"code": 20000, "data": f"【{sid}】删除成功！"}
|
||
else:
|
||
return {"code": 50000, "data": f"{reg.status_code}【{sid}】{reg.text}"}
|
||
|
||
def get_rules():
|
||
rules = """
|
||
- name: Domain
|
||
rules:
|
||
- alert: 站点可用性
|
||
expr: probe_success == 0
|
||
for: 1m
|
||
labels:
|
||
alertype: domain
|
||
severity: critical
|
||
annotations:
|
||
description: "{{ $labels.env }}_{{ $labels.name }}({{ $labels.project }})：站点无法访问\\n> {{ $labels.instance }}"
|
||
|
||
- alert: 站点1h可用性低于80%
|
||
expr: sum_over_time(probe_success[1h])/count_over_time(probe_success[1h]) * 100 < 80
|
||
for: 3m
|
||
labels:
|
||
alertype: domain
|
||
severity: warning
|
||
annotations:
|
||
description: "{{ $labels.env }}_{{ $labels.name }}({{ $labels.project }})：站点1h可用性：{{ $value | humanize }}%\\n> {{ $labels.instance }}"
|
||
|
||
- alert: 站点状态异常
|
||
expr: (probe_success == 0 and probe_http_status_code > 499) or probe_http_status_code == 0
|
||
for: 1m
|
||
labels:
|
||
alertype: domain
|
||
severity: warning
|
||
annotations:
|
||
description: "{{ $labels.env }}_{{ $labels.name }}({{ $labels.project }})：站点状态异常：{{ $value }}\\n> {{ $labels.instance }}"
|
||
|
||
- alert: 站点耗时过高
|
||
expr: probe_duration_seconds > 0.5
|
||
for: 2m
|
||
labels:
|
||
alertype: domain
|
||
severity: warning
|
||
annotations:
|
||
description: "{{ $labels.env }}_{{ $labels.name }}({{ $labels.project }})：当前站点耗时：{{ $value | humanize }}s\\n> {{ $labels.instance }}"
|
||
|
||
- alert: SSL证书有效期
|
||
expr: (probe_ssl_earliest_cert_expiry-time()) / 3600 / 24 < 15
|
||
for: 2m
|
||
labels:
|
||
alertype: domain
|
||
severity: warning
|
||
annotations:
|
||
description: "{{ $labels.env }}_{{ $labels.name }}({{ $labels.project }})：证书有效期剩余{{ $value | humanize }}天\\n> {{ $labels.instance }}"
|
||
"""
|
||
return {"code": 20000, "rules": rules}
|
||
|
||
def get_bconfig():
|
||
bconfig = """
|
||
modules:
|
||
http_2xx:
|
||
prober: http
|
||
http:
|
||
valid_status_codes: [200,204]
|
||
no_follow_redirects: false
|
||
preferred_ip_protocol: ip4
|
||
ip_protocol_fallback: false
|
||
|
||
# 用于需要检查SSL证书有效性，但是该域名访问后又会重定向到其它域名的情况，这样检查的证书有效期就是重定向后域名的。
|
||
# 如果需要检查源域名信息，需要在blackbox中增加禁止重定向参数。
|
||
httpNoRedirect4ssl:
|
||
prober: http
|
||
http:
|
||
valid_status_codes: [200,204,301,302,303]
|
||
no_follow_redirects: true
|
||
preferred_ip_protocol: ip4
|
||
ip_protocol_fallback: false
|
||
|
||
# 用于忽略SSL证书检查的站点监控。
|
||
http200igssl:
|
||
prober: http
|
||
http:
|
||
valid_status_codes:
|
||
- 200
|
||
tls_config:
|
||
insecure_skip_verify: true
|
||
|
||
http_4xx:
|
||
prober: http
|
||
http:
|
||
valid_status_codes: [401,403,404]
|
||
preferred_ip_protocol: ip4
|
||
ip_protocol_fallback: false
|
||
|
||
http_5xx:
|
||
prober: http
|
||
http:
|
||
valid_status_codes: [500,502]
|
||
preferred_ip_protocol: ip4
|
||
ip_protocol_fallback: false
|
||
|
||
http_post_2xx:
|
||
prober: http
|
||
http:
|
||
method: POST
|
||
|
||
icmp:
|
||
prober: icmp
|
||
|
||
tcp_connect:
|
||
prober: tcp
|
||
|
||
ssh_banner:
|
||
prober: tcp
|
||
tcp:
|
||
query_response:
|
||
- expect: "^SSH-2.0-"
|
||
- send: "SSH-2.0-blackbox-ssh-check"
|
||
"""
|
||
return {"code": 20000, "bconfig": bconfig}
|
||
|
||
def get_pconfig():
|
||
consul_server = consul_url.split("/")[2]
|
||
pconfig = f"""
|
||
- job_name: 'blackbox_exporter'
|
||
scrape_interval: 15s
|
||
scrape_timeout: 5s
|
||
metrics_path: /probe
|
||
consul_sd_configs:
|
||
- server: '{consul_server}'
|
||
token: '{consul_token}'
|
||
services: ['blackbox_exporter']
|
||
relabel_configs:
|
||
- source_labels: ["__meta_consul_service_metadata_instance"]
|
||
target_label: __param_target
|
||
- source_labels: [__meta_consul_service_metadata_module]
|
||
target_label: __param_module
|
||
- source_labels: [__meta_consul_service_metadata_module]
|
||
target_label: module
|
||
- source_labels: ["__meta_consul_service_metadata_company"]
|
||
target_label: company
|
||
- source_labels: ["__meta_consul_service_metadata_env"]
|
||
target_label: env
|
||
- source_labels: ["__meta_consul_service_metadata_name"]
|
||
target_label: name
|
||
- source_labels: ["__meta_consul_service_metadata_project"]
|
||
target_label: project
|
||
- source_labels: [__param_target]
|
||
target_label: instance
|
||
- target_label: __address__
|
||
replacement: 127.0.0.1:9115
|
||
"""
|
||
return {"code": 20000, "pconfig": pconfig}
|