ConsulManager/flask-consul/units/blackbox_manager.py

import requests,json,consul_kv
from config import consul_token,consul_url

headers = {'X-Consul-Token': consul_token}
init_module_list = ['http_2xx','http_4xx','tcp_connect','icmp','http200igssl','httpNoRedirect4ssl','http_5xx','http_post_2xx','ssh_banner']
def get_all_list(module,company,project,env):
    module = f'and Meta.module=="{module}"' if module != '' else f'and Meta.module != ""'
    company = f'and Meta.company=="{company}"' if company != '' else f'and Meta.company != ""'
    project = f'and Meta.project=="{project}"' if project != '' else f'and Meta.project != ""'
    env = f'and Meta.env=="{env}"' if env != '' else f'and Meta.env != ""'
    url = f'{consul_url}/agent/services?filter=Service == blackbox_exporter {module} {company} {project} {env}'
    response = requests.get(url, headers=headers)
    if response.status_code == 200:
        info = response.json()
        all_list = [i['Meta'] for i in info.values()]
        module_list = consul_kv.get_value('ConsulManager/record/blackbox/module_list')['module_list']
        company_list = sorted(list(set([i['company'] for i in all_list])))
        project_list = sorted(list(set([i['project'] for i in all_list])))
        env_list = sorted(list(set([i['env'] for i in all_list])))

        init_m_list = [x for x in init_module_list if x not in module_list]
        module_list = module_list + ['---'] + init_m_list

        return {'code': 20000,'all_list':all_list,'module_list':module_list,
                'company_list':company_list,'project_list':project_list,'env_list':env_list}
    else:
        return {'code': 50000, 'data': f'{response.status_code}:{response.text}'}

def get_service():
    response = requests.get(f'{consul_url}/agent/services?filter=Service == blackbox_exporter', headers=headers)
    if response.status_code == 200:
        info = response.json()
        all_list = [i['Meta'] for i in info.values()]
        module_list = sorted(list(set([i['module'] for i in all_list])))
        company_list = sorted(list(set([i['company'] for i in all_list])))
        project_list = sorted(list(set([i['project'] for i in all_list])))
        env_list = sorted(list(set([i['env'] for i in all_list])))
        consul_kv.put_kv('ConsulManager/record/blackbox/module_list',{'module_list':module_list})
        init_m_list = [x for x in init_module_list if x not in module_list]
        module_list = module_list + ['------'] + init_m_list

        return {'code': 20000,'all_list':all_list,'module_list':module_list,
                'company_list':company_list,'project_list':project_list,'env_list':env_list}
    else:
        return {'code': 50000, 'data': f'{response.status_code}:{response.text}'}

def add_service(module,company,project,env,name,instance):
    sid = f"{module}/{company}/{project}/{env}@{name}"
    if '//' in sid or sid.startswith('/') or sid.endswith('/'):
        return {"code": 50000, "data": f"服务ID【{sid}】首尾不能包含'/'，并且不能包含两个连续的'/'"}
    data = {
            "id": sid,
            "name": 'blackbox_exporter',
            "tags": [module],
            "Meta": {'module':module,'company':company,'project':project,'env':env,'name':name,'instance':instance}
           }
    reg = requests.put(f'{consul_url}/agent/service/register', headers=headers, data=json.dumps(data))
    if reg.status_code == 200:
        return {"code": 20000, "data": f"【{sid}】增加成功！"}
    else:
        return {"code": 50000, "data": f"{reg.status_code}【{sid}】{reg.text}"}

def del_service(module,company,project,env,name):
    sid = f"{module}/{company}/{project}/{env}@{name}"
    reg = requests.put(f'{consul_url}/agent/service/deregister/{sid}', headers=headers)
    if reg.status_code == 200:
        return {"code": 20000, "data": f"【{sid}】删除成功！"}
    else:
        return {"code": 50000, "data": f"{reg.status_code}【{sid}】{reg.text}"}

def get_rules():
    rules = """
- name: Domain
  rules:
  - alert: 站点可用性
    expr: probe_success{job="blackbox_exporter"} == 0
    for: 1m
    labels:
      alertype: domain
      severity: critical
    annotations:
      description: "{{ $labels.env }}_{{ $labels.name }}({{ $labels.project }})：站点无法访问\\n> {{ $labels.instance }}"

  - alert: 站点1h可用性低于80%
    expr: sum_over_time(probe_success{job="blackbox_exporter"}[1h])/count_over_time(probe_success{job="blackbox_exporter"}[1h]) * 100 < 80
    for: 3m
    labels:
      alertype: domain
      severity: warning
    annotations:
      description: "{{ $labels.env }}_{{ $labels.name }}({{ $labels.project }})：站点1h可用性：{{ $value | humanize }}%\\n> {{ $labels.instance }}"

  - alert: 站点状态异常
    expr: (probe_success{job="blackbox_exporter"} == 0 and probe_http_status_code > 499) or probe_http_status_code == 0
    for: 1m
    labels:
      alertype: domain
      severity: warning
    annotations:
      description: "{{ $labels.env }}_{{ $labels.name }}({{ $labels.project }})：站点状态异常：{{ $value }}\\n> {{ $labels.instance }}"

  - alert: 站点耗时过高
    expr: probe_duration_seconds > 0.5
    for: 2m
    labels:
      alertype: domain
      severity: warning
    annotations:
      description: "{{ $labels.env }}_{{ $labels.name }}({{ $labels.project }})：当前站点耗时：{{ $value | humanize }}s\\n> {{ $labels.instance }}"

  - alert: SSL证书有效期
    expr: (probe_ssl_earliest_cert_expiry-time()) / 3600 / 24 < 15
    for: 2m
    labels:
      alertype: domain
      severity: warning
    annotations:
      description: "{{ $labels.env }}_{{ $labels.name }}({{ $labels.project }})：证书有效期剩余{{ $value | humanize }}天\\n> {{ $labels.instance }}"
"""
    return {"code": 20000, "rules": rules}

def get_bconfig():
    bconfig = """
modules:
  http_2xx:
    prober: http
    http:
      valid_status_codes: [200,204]
      no_follow_redirects: false
      preferred_ip_protocol: ip4
      ip_protocol_fallback: false

  # 用于需要检查SSL证书有效性，但是该域名访问后又会重定向到其它域名的情况，这样检查的证书有效期就是重定向后域名的。
  # 如果需要检查源域名信息，需要在blackbox中增加禁止重定向参数。
  httpNoRedirect4ssl:
    prober: http
    http:
      valid_status_codes: [200,204,301,302,303]
      no_follow_redirects: true
      preferred_ip_protocol: ip4
      ip_protocol_fallback: false

  # 用于忽略SSL证书检查的站点监控。
  http200igssl:
    prober: http
    http:
      valid_status_codes:
      - 200
      tls_config:
        insecure_skip_verify: true

  http_4xx:
    prober: http
    http:
      valid_status_codes: [401,403,404]
      preferred_ip_protocol: ip4
      ip_protocol_fallback: false

  http_5xx:
    prober: http
    http:
      valid_status_codes: [500,502]
      preferred_ip_protocol: ip4
      ip_protocol_fallback: false

  http_post_2xx:
    prober: http
    http:
      method: POST

  icmp:
    prober: icmp

  tcp_connect:
    prober: tcp

  ssh_banner:
    prober: tcp
    tcp:
      query_response:
      - expect: "^SSH-2.0-"
      - send: "SSH-2.0-blackbox-ssh-check"
"""
    return {"code": 20000, "bconfig": bconfig}

def get_pconfig():
    consul_server = consul_url.split("/")[2]
    pconfig = f"""
  - job_name: 'blackbox_exporter'
    scrape_interval: 15s
    scrape_timeout: 5s
    metrics_path: /probe
    consul_sd_configs:
      - server: '{consul_server}'
        token: '{consul_token}'
        services: ['blackbox_exporter']
    relabel_configs:
      - source_labels: ["__meta_consul_service_metadata_instance"]
        target_label: __param_target
      - source_labels: [__meta_consul_service_metadata_module]
        target_label: __param_module
      - source_labels: [__meta_consul_service_metadata_module]
        target_label: module
      - source_labels: ["__meta_consul_service_metadata_company"]
        target_label: company
      - source_labels: ["__meta_consul_service_metadata_env"]
        target_label: env
      - source_labels: ["__meta_consul_service_metadata_name"]
        target_label: name
      - source_labels: ["__meta_consul_service_metadata_project"]
        target_label: project
      - source_labels: [__param_target]
        target_label: instance
      - target_label: __address__
        replacement: 127.0.0.1:9115
"""
    return {"code": 20000, "pconfig": pconfig}