add clickhouse jms

flask-consul/requirements.txt

@@ -27,6 +27,8 @@ alibabacloud_bssopenapi20171214==2.0.6
 aliyun-python-sdk-cms==7.0.32
 alibabacloud_dds20151201==8.1.1
 alibabacloud_polardb20170801==5.4.0
+alibabacloud_clickhouse20191111==3.1.12
+alibabacloud_clickhouse20230522==1.0.2
 tencentcloud-sdk-python-common==3.0.779
 tencentcloud-sdk-python-cvm==3.0.779
 tencentcloud-sdk-python-cdb==3.0.779

flask-consul/units/cloud/alicloud.py

@@ -17,10 +17,14 @@ from alibabacloud_polardb20170801.client import Client as polardb20170801Client
 from alibabacloud_polardb20170801 import models as polardb_20170801_models
 from alibabacloud_dds20151201.client import Client as Dds20151201Client
 from alibabacloud_dds20151201 import models as dds_20151201_models
+from alibabacloud_clickhouse20191111.client import Client as clickhouse20191111Client
+from alibabacloud_clickhouse20191111 import models as clickhouse_20191111_models
+from alibabacloud_clickhouse20230522.client import Client as clickhouse20230522Client
+from alibabacloud_clickhouse20230522 import models as clickhouse_20230522_models
 
 import sys,datetime,hashlib,math,traceback
 from units import consul_kv,consul_svc
-from units.cloud import sync_ecs,sync_rds,sync_redis,notify,sync_polardb,sync_mongodb
+from units.cloud import sync_ecs,sync_rds,sync_redis,notify,sync_polardb,sync_mongodb,sync_clickhouse
 from units.config_log import *
 
 def exp(account,collect_days,notify_days,notify_amount):
@@ -396,7 +400,6 @@ def polardb(account, region):
             else:
                 page_number += 1
         try:
-            print("-------------1", polardb_dict)
             for iid in polardb_dict.keys():
                 logger.info(f'【ali_PolarDB】===> {iid}')
                 describe_dbcluster_attribute_request = polardb_20170801_models.DescribeDBClusterEndpointsRequest(
@@ -535,4 +538,121 @@ def mongodb(account, region):
     except Exception as e:
         logger.error(f'{e}\n{traceback.format_exc()}')
         data = {'count': '无', 'update': '失败', 'status': 50000, 'msg': str(e)}
-        consul_kv.put_kv(f'ConsulManager/record/jobs/alicloud/{account}/mongodb/{region}', data)
+        consul_kv.put_kv(f'ConsulManager/record/jobs/alicloud/{account}/mongodb/{region}', data)
+
+
+def clickhouse(account, region):
+    ak, sk = consul_kv.get_aksk('alicloud', account)
+    now = datetime.datetime.now().strftime('%m.%d/%H:%M')
+    group_dict = consul_kv.get_value(f'ConsulManager/assets/alicloud/group/{account}')
+
+    config = open_api_models.Config(access_key_id=ak, access_key_secret=sk)
+    config.endpoint = 'clickhouse.aliyuncs.com'
+    if region == "ap-southeast-1":
+        config.endpoint = f'clickhouse.{region}.aliyuncs.com'
+    elif region == "eu-central-1":
+        config.endpoint = f'clickhouse.{region}.aliyuncs.com'
+    elif region == "us-west-1":
+        config.endpoint = f'clickhouse.{region}.aliyuncs.com'
+    client = clickhouse20191111Client(config)
+    page_number = 1
+    clickhouse_dict = {}
+    runtime = util_models.RuntimeOptions()
+    try:
+        while True:
+            describe_dbinstances_request = clickhouse_20191111_models.DescribeDBClustersRequest(
+                page_size=100,
+                page_number=page_number,
+                region_id=region
+            )
+            clickhouse_info = client.describe_dbclusters_with_options(describe_dbinstances_request, runtime)
+            clickhouse_list = clickhouse_info.body.to_map()['DBClusters']["DBCluster"]
+
+            clickhouse_dict_temp = {i['DBClusterId']: {
+                'name': i.get('DBClusterDescription', f"未命名{i['DBClusterId']}"),
+                'domain': i['ConnectionString'],
+                'ip': '无',
+                'port': '9000',
+                'region': region,
+                'group': group_dict.get(i['ResourceGroupId'], '无'),
+                'status': i['DBClusterStatus'],
+                'itype': '社区版',
+                'ver': i['DbVersion'],
+                'exp': '-' if i.get('ExpireTime', None) is None else i.get('ExpireTime', '-T').split('T')[0],
+                'cpu': '无', 'mem': '无', 'disk': '无'
+            } for i in clickhouse_list}
+            clickhouse_dict.update(clickhouse_dict_temp)
+            if len(clickhouse_list) < 100:
+                break
+            else:
+                page_number += 1
+
+        client = clickhouse20230522Client(config)
+        page_number = 1
+        while True:
+            describe_dbinstances_request = clickhouse_20230522_models.DescribeDBInstancesRequest(
+                page_size=100,
+                page_number=page_number,
+                region_id=region
+            )
+            clickhouse_info = client.describe_dbinstances_with_options(describe_dbinstances_request, runtime)
+            clickhouse_list = clickhouse_info.body.to_map()['Data']["DBInstances"]
+
+            clickhouse_dict_temp = {i['DBInstanceId']: {
+                'name': i.get('Description', f"未命名{i['DBInstanceId']}"),
+                'domain': '无',
+                'ip': '无',
+                'port': '9000',
+                'region': region,
+                'group': group_dict.get(i['ResourceGroupId'], '无'),
+                'status': i['Status'],
+                'itype': '企业版',
+                'ver': i['EngineVersion'],
+                'exp': '-' if i.get('ExpireTime', None) is None else i.get('ExpireTime', '-T').split('T')[0],
+                'cpu': '无', 'mem': '无', 'disk': '无'
+            } for i in clickhouse_list}
+            clickhouse_dict.update(clickhouse_dict_temp)
+            if len(clickhouse_list) < 100:
+                break
+            else:
+                page_number += 1
+        try:
+            for iid in clickhouse_dict.keys():
+                logger.info(f'【ali_Clickhouse】===> {iid}')
+                if clickhouse_dict[iid].get('domain') == '无':
+                    describe_dbinstance_attribute_request = clickhouse_20230522_models.DescribeEndpointsRequest(region_id=region, dbinstance_id=iid)
+                    clickhouse_plus_info = client.describe_endpoints_with_options(describe_dbinstance_attribute_request, runtime)
+                    clickhouse_plus_list = clickhouse_plus_info.body.to_map()['Data']["Endpoints"]
+                    clickhouse_plus= {}
+                    for i in clickhouse_plus_list:
+                        if i['NetType'] == 'VPC':
+                            clickhouse_plus[iid] = {
+                                'domain': i['VpcInstanceId'],
+                                'ip': i['IPAddress'],
+                            }
+                            break
+                    for k, v in clickhouse_plus.items():
+                        if k in clickhouse_dict:
+                            clickhouse_dict[k].update(v)
+        except Exception as e:
+            logger.error('DescribeDBInstancesAsCsvRequest ERROR' + f'{e}\n{traceback.format_exc()}')
+        count = len(clickhouse_dict)
+        off, on = sync_clickhouse.w2consul('alicloud', account, region, clickhouse_dict)
+        data = {'count': count, 'update': now, 'status': 20000, 'on': on, 'off': off,
+                'msg': f'clickhouse同步成功！总数：{count}，开机：{on}，关机：{off}'}
+        consul_kv.put_kv(f'ConsulManager/record/jobs/alicloud/{account}/clickhouse/{region}', data)
+        logger.info(f'【JOB】===>alicloud_clickhouse {account} {region} {data}')
+    except TeaException as e:
+        emsg = e.message.split('. ', 1)[0]
+        logger.error(f"【code:】{e.code}\n【message:】{e.message}\n{traceback.format_exc()}")
+        data = consul_kv.get_value(f'ConsulManager/record/jobs/alicloud/{account}/clickhouse/{region}')
+        if data == {}:
+            data = {'count': '无', 'update': f'失败{e.code}', 'status': 50000, 'msg': emsg}
+        else:
+            data['update'] = f'失败{e.code}'
+            data['msg'] = emsg
+        consul_kv.put_kv(f'ConsulManager/record/jobs/alicloud/{account}/clickhouse/{region}', data)
+    except Exception as e:
+        logger.error(f'{e}\n{traceback.format_exc()}')
+        data = {'count': '无', 'update': '失败', 'status': 50000, 'msg': str(e)}
+        consul_kv.put_kv(f'ConsulManager/record/jobs/alicloud/{account}/clickhouse/{region}', data)

flask-consul/units/cloud/sync_clickhouse.py

@@ -0,0 +1,77 @@
+#!/usr/bin/python3
+import requests,json
+from units import consul_kv
+from config import consul_token,consul_url,vendors,regions
+from units.config_log import *
+headers = {'X-Consul-Token': consul_token}
+geturl = f'{consul_url}/agent/services'
+delurl = f'{consul_url}/agent/service/deregister'
+puturl = f'{consul_url}/agent/service/register'
+def w2consul(vendor,account,region,clickhouse_dict):
+    service_name = f'{vendor}_{account}_clickhouse'
+    params = {'filter': f'Service == "{service_name}" and "{region}" in Tags and Meta.account == "{account}"'}
+    try:
+        consul_clickhouse_iid_list = requests.get(geturl, headers=headers, params=params).json().keys()
+    except:
+        consul_clickhouse_iid_list = []
+        
+    #在consul中删除云厂商不存在的clickhouse
+    for del_clickhouse in [x for x in consul_clickhouse_iid_list if x not in clickhouse_dict.keys()]:
+        dereg = requests.put(f'{delurl}/{del_clickhouse}', headers=headers)
+        if dereg.status_code == 200:
+            logger.info(f"code: 20000, data: {account}-删除成功！")
+        else:
+            logger.info(f"code: 50000, data: {dereg.status_code}:{dereg.text}")
+    off,on = 0,0
+    for k,v in clickhouse_dict.items():
+        iid = k
+        #对consul中关机的clickhouse做标记。
+        if v['status'] in ['SHUTDOWN','非运行中']:
+            off = off + 1
+            tags = ['OFF',v['itype'],v['ver'], region]
+            stat = 'off'
+        else:
+            on = on + 1
+            tags = ['ON',v['itype'],v['ver'],region]
+            stat = 'on'
+        custom_clickhouse = consul_kv.get_value(f'ConsulManager/assets/sync_clickhouse_custom/{iid}')
+        port = custom_clickhouse.get('port')
+        ip = custom_clickhouse.get('ip')
+        if port == None:
+            port = v['port']
+        if ip == None:
+            ip = v['ip']
+        instance = f'{ip}:{port}'
+        data = {
+            'id': iid,
+            'name': service_name,
+            'Address': ip,
+            'port': port,
+            'tags': tags,
+            'Meta': {
+                'iid': iid,
+                'name': v['name'],
+                'region': regions[vendor].get(region,'未找到'),
+                'group': v['group'],
+                'instance': instance,
+                'account': account,
+                'itype': v['itype'],
+                'vendor': vendors.get(vendor,'未找到'),
+                'os': "clickhouse",
+                'ver': v['ver'],
+                'domain':v['domain'],
+                'exp': v['exp'],
+                'stat': stat,
+                'team': v.get('team','无')
+            },
+            "check": {
+                "tcp": f"{ip}:{port}",
+                "interval": "60s"
+            }
+        }
+        reg = requests.put(puturl, headers=headers, data=json.dumps(data))
+        if reg.status_code == 200:
+            pass
+        else:
+            logger.info(f"{account}:code: 5000, data: {reg.status_code}:{reg.text}")
+    return off,on

flask-consul/units/jms/sync_jms.py

@@ -2,7 +2,7 @@ import datetime,requests,json,traceback
 from units import consul_kv,consul_manager,myaes
 from units.config_log import *
 
-resource_type = ["ecs", "redis", "rds", "mongodb", "polardb"]
+resource_type = ["ecs", "redis", "rds", "mongodb", "polardb", "clickhouse"]
 
 
 # 创建节点
@@ -110,19 +110,24 @@ def update_jms(jms_ver,jms_url,headers,new_node_dict,node_id,cloud,account,ecs_i
             ecs_url = f"{jms_url}/api/v1/assets/databases/"
             payload["protocols"] = [{"name": proto, "port": port}]
             payload.update({"db_name": "0", "use_ssl": False, "allow_invalid_cert": False})
-            if float(v['ver']) < 6:
-                payload["platform"] = '24'
-            else:
-                payload["platform"] = "25"
+            # if float(v['ver']) < 6:
+            #     payload["platform"] = '24'
+            # else:
+            payload["platform"] = "25"
         elif platform == 'Mysql':
             ecs_url = f"{jms_url}/api/v1/assets/databases/"
             payload["platform"] = '17'
             payload["protocols"] = [{"name": proto, "port": port}]
         elif platform == 'Mongodb':
             ecs_url = f"{jms_url}/api/v1/assets/databases/"
-            payload["platform"] = '23'
+            payload["platform"] = '33'
             payload["protocols"] = [{"name": proto, "port": port}]
             payload.update({"db_name": "admin"})
+        elif platform == 'Clickhouse':
+            ecs_url = f"{jms_url}/api/v1/assets/databases/"
+            payload["platform"] = '22'
+            payload["protocols"] = [{"name": proto, "port": port}]
+            payload.update({"db_name": "default"})
         else:
             logger.error(f"未匹配到{platform}！")
             continue

flask-consul/units/selfclickhouse_manager.py

@@ -0,0 +1,77 @@
+import requests,json,re
+from config import consul_token,consul_url
+from units.config_log import *
+headers = {'X-Consul-Token': consul_token}
+
+def get_all_list(vendor,account,region,group):
+    vendor = f'and Meta.vendor=="{vendor}"' if vendor != '' else f'and Meta.vendor != ""'
+    account = f'and Meta.account=="{account}"' if account != '' else f'and Meta.account != ""'
+    region = f'and Meta.region=="{region}"' if region != '' else f'and Meta.region != ""'
+    group = f'and Meta.group=="{group}"' if group != '' else f'and Meta.group != ""'
+    url = f'{consul_url}/agent/services?filter=Service == selfrds_exporter {vendor} {account} {region} {group}'
+    response = requests.get(url, headers=headers)
+    if response.status_code == 200:
+        info = response.json()
+        all_list = [i['Meta'] for i in info.values()]
+        vendor_list = sorted(list(set([i['vendor'] for i in all_list])))
+        account_list = sorted(list(set([i['account'] for i in all_list])))
+        region_list = sorted(list(set([i['region'] for i in all_list])))
+        group_list = sorted(list(set([i['group'] for i in all_list])))
+        return {'code': 20000,'all_list':all_list,'vendor_list':vendor_list,
+                'account_list':account_list,'region_list':region_list,'group_list':group_list}
+    else:
+        logger.error(f"{response.status_code}：{response.text}")
+        return {'code': 50000, 'data': f'{response.status_code}:{response.text}'}
+
+def get_service():
+    response = requests.get(f'{consul_url}/agent/services?filter=Service == selfrds_exporter', headers=headers)
+    if response.status_code == 200:
+        info = response.json()
+        all_list = [i['Meta'] for i in info.values()]
+        vendor_list = sorted(list(set([i['vendor'] for i in all_list])))
+        account_list = sorted(list(set([i['account'] for i in all_list])))
+        region_list = sorted(list(set([i['region'] for i in all_list])))
+        group_list = sorted(list(set([i['group'] for i in all_list])))
+        return {'code': 20000,'all_list':all_list,'vendor_list':vendor_list,
+                'account_list':account_list,'region_list':region_list,'group_list':group_list}
+    else:
+        logger.error(f"{response.status_code}：{response.text}")
+        return {'code': 50000, 'data': f'{response.status_code}:{response.text}'}
+
+def add_service(vendor,account,region,group,name,ip,port,os):
+    if port is None or name is None:
+        return {"code": 50000, "data": f"名称或IP不能为空！"}
+    sid = f"{vendor}/{account}/{region}/{group}@{name}".strip()
+    #sid = eval(repr(sid).replace('\\t','').replace('\\n',''))
+    sid = re.sub('[[ \]`~!\\\#$^&*=|"{}\':;?\t\n]','_',sid)+'@rds'
+    instance = f'{ip}:{port}'
+    if '//' in sid or sid.startswith('/'):
+        return {"code": 50000, "data": f"服务ID【{sid}】首尾不能包含'/'，并且不能包含两个连续的'/'"}
+    data = {
+            "id": sid,
+            "name": 'selfrds_exporter',
+            'Address': ip,
+            'port': int(port),
+            "tags": [vendor,os],
+            "Meta": {'vendor':vendor,'account':account,'region':region,'group':group,
+                     'name':name,'instance':instance,'os':os},
+            "check": {"tcp": instance,"interval": "60s"}
+           }
+    reg = requests.put(f'{consul_url}/agent/service/register', headers=headers, data=json.dumps(data))
+    if reg.status_code == 200:
+        return {"code": 20000, "data": f"【{sid}】增加成功！"}
+    else:
+        logger.error(f"{reg.status_code}【{sid}】{reg.text}")
+        return {"code": 50000, "data": f"{reg.status_code}【{sid}】{reg.text}"}
+
+def del_service(vendor,account,region,group,name):
+    sid = f"{vendor}/{account}/{region}/{group}@{name}".strip()
+    sid = re.sub('[[ \]`~!\\\#$^&*=|"{}\':;?\t\n]','_',sid)+'@rds'
+    reg = requests.put(f'{consul_url}/agent/service/deregister/{sid}', headers=headers)
+    if reg.status_code == 200:
+        logger.debug(f"【{sid}】删除成功！")
+        return {"code": 20000, "data": f"【{sid}】删除成功！"}
+    else:
+        logger.error(f"{reg.status_code}【{sid}】{reg.text}")
+        return {"code": 50000, "data": f"{reg.status_code}【{sid}】{reg.text}"}
+

flask-consul/views/clickhouse.py

@@ -0,0 +1,93 @@
+from flask import Blueprint
+from flask_restful import reqparse, Resource, Api
+from flask_apscheduler import APScheduler
+import traceback
+#import sys
+#sys.path.append("..")
+from units import token_auth,consul_kv,gen_config,consul_svc
+from units.config_log import *
+blueprint = Blueprint('clickhouse',__name__)
+api = Api(blueprint)
+
+parser = reqparse.RequestParser()
+parser.add_argument('job_id',type=str)
+parser.add_argument('services_dict',type=dict)
+parser.add_argument('cst_clickhouse_dict',type=dict)
+parser.add_argument('iid',type=str)
+parser.add_argument('jobclickhouse_name',type=str)
+parser.add_argument('checked',type=str)
+
+class Clickhouse(Resource):
+    decorators = [token_auth.auth.login_required]
+    def get(self, stype):
+        job_id = parser.parse_args()['job_id']
+        if stype == 'jobclickhouse':
+            jobclickhouse = consul_kv.get_keys_list('ConsulManager/jobs')
+            jobclickhouse_list = [i.split('/jobs/')[1] for i in jobclickhouse if '/clickhouse/' in i]
+            return {'code': 20000,'jobclickhouse':jobclickhouse_list}
+        elif stype == 'clickhouse_services':
+            jobecs = consul_kv.get_keys_list('ConsulManager/jobs')
+            jobecs_list = [i.split('/jobs/')[1] for i in jobecs if '/clickhouse/' in i]
+            services_list = []
+            for i in jobecs_list:
+                serivces = i.split("/")
+                services_list.append(f'{serivces[0]}_{serivces[1]}_{serivces[2]}')
+            return {'code': 20000,'services_list': sorted(set(services_list))}
+        elif stype == 'clickhouserules':
+            return gen_config.get_clickhouserules()
+        elif stype == 'cstclickhouseconf':
+            args = parser.parse_args()
+            iid = args['iid']
+            cst_clickhouse_config = consul_kv.get_value(f'ConsulManager/assets/sync_clickhouse_custom/{iid}')
+            cst_clickhouse_config.update({'iid': iid,'ipswitch': False,'portswitch': False})
+            if 'ip' in cst_clickhouse_config and cst_clickhouse_config['ip'] != '':
+                cst_clickhouse_config['ipswitch'] = True
+            if 'port' in cst_clickhouse_config and cst_clickhouse_config['port'] != '':
+                cst_clickhouse_config['portswitch'] = True
+            return {'code': 20000, 'cst_clickhouse': cst_clickhouse_config}
+        elif stype == 'cstclickhouselist':
+            args = parser.parse_args()
+            jobclickhouse_name = args['jobclickhouse_name']
+            checked = args['checked']
+            cst_clickhouse_dict = consul_kv.get_kv_dict('ConsulManager/assets/sync_clickhouse_custom/')
+            cst_clickhouse_keylist = [k.split('/')[-1] for k,v in cst_clickhouse_dict.items() if v != {}]
+            clickhouse_info = consul_kv.get_res_services(jobclickhouse_name)
+            if checked == 'false':
+                return clickhouse_info
+            else:
+                cst_clickhouse_list = [i for i in clickhouse_info['res_list'] if i['iid'] in cst_clickhouse_keylist]
+                return {'code': 20000, 'res_list': cst_clickhouse_list}
+                
+    def post(self, stype):
+        if stype == 'clickhousepconfig':
+            args = parser.parse_args()
+            services_dict = args['services_dict']
+            return gen_config.clickhouse_config(services_dict['jobclickhouse_list'],services_dict['cm_exporter'],services_dict['services_list'],services_dict['exporter'])
+        elif stype == 'cstclickhouse':
+            args = parser.parse_args()
+            cst_clickhouse_dict = args['cst_clickhouse_dict']
+            consul_clickhouse_cst = {}
+            iid = cst_clickhouse_dict['iid']
+            try:
+                sid_dict = consul_svc.get_sid(iid)['instance']
+                if cst_clickhouse_dict['portswitch'] and cst_clickhouse_dict['port'] != '':
+                    consul_clickhouse_cst['port'] = int(cst_clickhouse_dict['port'])
+                    sid_dict['Port'] = consul_clickhouse_cst['port']
+                if cst_clickhouse_dict['ipswitch'] and cst_clickhouse_dict['ip'] != '':
+                    consul_clickhouse_cst['ip'] = cst_clickhouse_dict['ip']
+                    sid_dict['Address'] = consul_clickhouse_cst['ip']
+                consul_kv.put_kv(f'ConsulManager/assets/sync_clickhouse_custom/{iid}',consul_clickhouse_cst)
+                del sid_dict['Weights']
+                del sid_dict['ContentHash']
+                del sid_dict['Datacenter']
+                sid_dict['name'] = sid_dict.pop('Service')
+                sid_dict['Meta']['instance'] = f"{sid_dict['Address']}:{sid_dict['Port']}"
+                sid_dict["check"] = { "tcp": sid_dict['Meta']['instance'],"interval": "60s" }
+                consul_svc.del_sid(iid)
+                consul_svc.add_sid(sid_dict)
+                return {'code': 20000, 'data': '自定义实例信息修改成功！'}
+            except Exception as e:
+                logger.error(f'{e}\n{traceback.format_exc()}')
+                return {'code': 50000, "data": '提交自定义实例信息格式错误！'}
+
+api.add_resource(Clickhouse, '/api/clickhouse/<stype>')

flask-consul/views/edit_cloud.py

@@ -35,7 +35,7 @@ class Edit(Resource):
             region = args['region']
             restype = ['group']
             interval = {'proj_interval': 60, 'ecs_interval': 10, 'rds_interval': 20, 'redis_interval': 20,
-                        'polardb_interval': 20, 'mongodb_interval': 20}
+                        'polardb_interval': 20, 'mongodb_interval': 20, 'clickhouse_interval': 20}
             isextip = False
             for i in job_list:
                 if f'{vendor}/{account}/group' == i['id']:
@@ -56,6 +56,9 @@ class Edit(Resource):
                 elif f'{vendor}/{account}/mongodb/{region}' == i['id']:
                     restype.append('mongodb')
                     interval['mongodb_interval'] = i['minutes']
+                elif f'{vendor}/{account}/clickhouse/{region}' == i['id']:
+                    restype.append('clickhouse')
+                    interval['clickhouse_interval'] = i['minutes']
             return {'code': 20000, 'restype': restype, 'interval': interval, 'isextip': isextip}
     def post(self,stype):
         job_list = list(consul_kv.get_kv_dict(f'ConsulManager/jobs').values())
@@ -73,6 +76,7 @@ class Edit(Resource):
             polardb_interval = int(editjob_dict['polardb_interval'])
             mongodb_interval = int(editjob_dict['mongodb_interval'])
             redis_interval = int(editjob_dict['redis_interval'])
+            clickhouse_interval = int(editjob_dict['clickhouse_interval'])
             logger.info(f'{editjob_dict}')
             if editjob_dict['akskswitch']:
                 ak = editjob_dict['ak']
@@ -91,6 +95,7 @@ class Edit(Resource):
             polardb_jobid = f'{vendor}/{account}/polardb/{region}'
             mongodb_jobid = f'{vendor}/{account}/mongodb/{region}'
             redis_jobid = f'{vendor}/{account}/redis/{region}'
+            clickhouse_jobid = f'{vendor}/{account}/clickhouse/{region}'
             if 'ecs' in restype:
                 isecs = [x for x in job_list if x['id'] == f'{vendor}/{account}/ecs/{region}']
                 if len(isecs) == 1:
@@ -209,5 +214,27 @@ class Edit(Resource):
                 except:
                     pass
 
+            if 'clickhouse' in restype:
+                isclickhouse = [x for x in job_list if x['id'] == f'{vendor}/{account}/clickhouse/{region}']
+                if len(isclickhouse) == 1:
+                    if clickhouse_interval != isclickhouse[0]['minutes']:
+                        isclickhouse[0]['minutes'] = clickhouse_interval
+                        consul_kv.put_kv(f'ConsulManager/jobs/{clickhouse_jobid}',isclickhouse[0])
+                        modjob_interval(clickhouse_jobid,clickhouse_interval)
+                else:
+                    job_func = f"__main__:{vendor}.clickhouse"
+                    job_args = [account,region]
+                    job_interval = clickhouse_interval
+                    addjob(clickhouse_jobid, job_func, job_args, job_interval)
+                    job_dict = {'id':clickhouse_jobid,'func':job_func,'args':job_args,'minutes':job_interval,
+                                "trigger": "interval","replace_existing": True}
+                    consul_kv.put_kv(f'ConsulManager/jobs/{clickhouse_jobid}',job_dict)
+            else:
+                try:
+                    consul_kv.del_key(f'ConsulManager/jobs/{clickhouse_jobid}')
+                    deljob(clickhouse_jobid)
+                except:
+                    pass
+
             return {'code': 20000, 'data': f'{vendor}/{account}/{region}：编辑成功！'}
 api.add_resource(Edit, '/api/edit/<stype>')

flask-consul/views/jms.py

@@ -45,7 +45,7 @@ class Jms(Resource):
                 services_meta = consul_kv.get_services_meta(f'{vendor}_{account}_ecs').get('ecs_list', [])
                 count_ecs = len(services_meta)
                 count_off, count_on, count_cpu, count_mem, count_win, count_linux = 0, 0, 0, 0, 0, 0
-                count_redis, count_mongodb, count_mysql = 0, 0, 0
+                count_redis, count_mongodb, count_mysql, count_clickhouse = 0, 0, 0, 0
                 for i in services_meta:
                     if i['os'] == 'linux':
                         count_linux = count_linux + 1
@@ -65,6 +65,8 @@ class Jms(Resource):
                         count_mysql = count_mysql + 1
                     elif i['os'] == 'mongodb':
                         count_mongodb = count_mongodb + 1
+                    elif i['os'] == 'clickhouse':
+                        count_clickhouse = count_clickhouse + 1
                     if i.get('stat') == 'off':
                         count_off = count_off + 1
                     else:
@@ -89,6 +91,7 @@ class Jms(Resource):
                         'count_redis': count_redis,
                         'count_mysql': count_mysql,
                         'count_mongodb': count_mongodb,
+                        'count_clickhouse': count_clickhouse,
                         'count_mem': f'{count_mem}GB',
                         'count_cpu': f'{count_cpu}核',
                         'count_ecs': count_ecs,
@@ -119,6 +122,8 @@ class Jms(Resource):
                 mysqluid = ecs_info['mysql'][-1]
                 mongodbport = ecs_info['mongodb'][0][0].split('/')[-1]
                 mongodbuid = ecs_info['mongodb'][-1]
+                clickhouseport = ecs_info['clickhouse'][0][0].split('/')[-1]
+                clickhouseuid = ecs_info['clickhouse'][-1]
 
                 token = myaes.decrypt(jms_info['token'])
                 custom_ecs_json = json.dumps(custom_ecs_info, indent=8) if custom_ecs_info != {} else ''
@@ -136,6 +141,8 @@ class Jms(Resource):
                     'mysqluid': mysqluid,
                     'mongodbport': mongodbport,
                     'mongodbuid': mongodbuid,
+                    'clickhouseport': clickhouseport,
+                    'clickhouseuid': clickhouseuid,
                     'custom_ecs_info': custom_ecs_json,
                 }
             else:
@@ -155,6 +162,7 @@ class Jms(Resource):
                 "redis": [[f"redis/{jms_config['redisport']}"], jms_config['redisuid']],
                 "mysql": [[f"mysql/{jms_config['mysqlport']}"], jms_config['mysqluid']],
                 "mongodb": [[f"mongodb/{jms_config['mongodbport']}"], jms_config['mongodbuid']],
+                "clickhouse": [[f"clickhouse/{jms_config['clickhouseport']}"], jms_config['clickhouseuid']],
             }
             consul_kv.put_kv('ConsulManager/jms/ecs_info', ecs_info)
             custom_ecs_info = jms_config['custom_ecs_info']

flask-consul/views/selfclickhouse.py

@@ -0,0 +1,80 @@
+import os
+from flask import Blueprint
+from flask_restful import reqparse, Resource, Api
+import sys,traceback
+sys.path.append("..")
+from units import token_auth,selfclickhouse_manager
+from werkzeug.datastructures import FileStorage
+from units import upload
+from units.config_log import *
+blueprint = Blueprint('selfclickhouse',__name__)
+api = Api(blueprint)
+
+parser = reqparse.RequestParser()
+parser.add_argument('vendor',type=str)
+parser.add_argument('account',type=str)
+parser.add_argument('region',type=str)
+parser.add_argument('group',type=str)
+parser.add_argument('name',type=str)
+parser.add_argument('ip',type=str)
+parser.add_argument('port',type=str)
+parser.add_argument('os',type=str)
+parser.add_argument('del_dict',type=dict)
+parser.add_argument('up_dict',type=dict)
+parser.add_argument('file',type=FileStorage, location="files", help="File is wrong.")
+
+class Upload(Resource):
+    @token_auth.auth.login_required
+    def post(self):
+        file = parser.parse_args().get("file")
+        try:
+            filename = file.filename
+            file_extension = os.path.splitext(filename)[1].lower()
+            try:
+                file_data = file.read()
+                if file_extension == '.xlsx':
+                    return upload.read_execl(file_data,'selfclickhouse')
+                elif file_extension == '.csv':
+                    return upload.read_csv(file_data,'selfclickhouse')
+            except Exception as e:
+                logger.error(f"【selfclickhouse】文件后缀名错误，请导入xlsx或csv格式,{e}\n{traceback.format_exc()}")
+                return {"code": 50000, "data": f"文件后缀名错误，请导入xlsx或csv格式！"}
+        except Exception as e:
+            logger.error(f"【selfclickhouse】导入失败,{e}\n{traceback.format_exc()}")
+            return {"code": 50000, "data": f"导入失败！"}
+
+class GetAllList(Resource):
+    @token_auth.auth.login_required
+    def get(self):
+        args = parser.parse_args()
+        return selfclickhouse_manager.get_all_list(args['vendor'],args['account'],args['region'],args['group'])
+
+class SelfclickhouseApi(Resource):
+    decorators = [token_auth.auth.login_required]
+    def get(self):
+        return selfclickhouse_manager.get_service()
+    def post(self):
+        args = parser.parse_args()
+        logger.info(f'=======\n{args}')
+        return selfclickhouse_manager.add_service(args['vendor'],args['account'],args['region'],
+                                            args['group'],args['name'],args['ip'],args['port'],args['os'])
+    def put(self):
+        args = parser.parse_args()
+        del_dict = args['del_dict']
+        up_dict = args['up_dict']
+        resp_del = selfclickhouse_manager.del_service(del_dict['vendor'],del_dict['account'],
+                                                del_dict['region'],del_dict['group'],del_dict['name'])
+        resp_add = selfclickhouse_manager.add_service(up_dict['vendor'],up_dict['account'],up_dict['region'],
+                                                up_dict['group'],up_dict['name'],up_dict['ip'],
+                                                up_dict['port'],up_dict['os'])
+        if resp_del["code"] == 20000 and resp_add["code"] == 20000:
+            return {"code": 20000, "data": f"更新成功！"}
+        else:
+            return {"code": 50000, "data": f"更新失败！"}
+    def delete(self):
+        args = parser.parse_args()
+        return selfclickhouse_manager.del_service(args['vendor'],args['account'],args['region'],args['group'],args['name'])
+
+api.add_resource(GetAllList,'/api/selfclickhouse/alllist')
+api.add_resource(SelfclickhouseApi, '/api/selfclickhouse/service')
+api.add_resource(Upload,'/api/selfclickhouse/upload')

vue-consul/src/views/jms/index.vue

@@ -67,6 +67,11 @@
           mongodb端口：<el-input v-model="jms_config.mongodbport" style="width: 72px;" />
           &nbsp;&nbsp;管理用户ID：<el-input v-model="jms_config.mongodbuid" style="width: 300px;" />
         </div>
+        <div class="demo-input-suffix">
+          <h4>Clickhouse：</h4>
+          clickhouse端口：<el-input v-model="jms_config.clickhouseport" style="width: 72px;" />
+          &nbsp;&nbsp;管理用户ID：<el-input v-model="jms_config.clickhouseuid" style="width: 300px;" />
+        </div>
         <div class="demo-input-suffix">
           <h4>Mysql：</h4>
           mysql端口：<el-input v-model="jms_config.mysqlport" style="width: 72px;" />
@@ -95,6 +100,7 @@
       <el-table-column prop="count_redis" label="Redis" sortable align="center" width="80" />
       <el-table-column prop="count_mysql" label="Mysql" sortable align="center" width="80" />
       <el-table-column prop="count_mongodb" label="Mongodb" sortable align="center" width="80" />
+      <el-table-column prop="count_clickhouse" label="Clickhouse" sortable align="center" width="80" />
       <el-table-column prop="count_mem" label="总内存" sortable align="center" width="120" />
       <el-table-column prop="count_cpu" label="总CPU" sortable align="center" width="110" />
       <el-table-column prop="count_ecs" label="资源数" sortable align="center" width="100">
@@ -149,7 +155,8 @@ export default {
   data() {
     return {
       jms_config: { ver: 'V2', url: '', token: '', linuxport: '22', linuxuid: '', winport: '3389', winuid: '',
-        redisport: '6379', redisuid: '', mysqlport: '3306', mysqluid: '', mongodbport: '3717', mongodbuid: '', custom_ecs_info: '' },
+        redisport: '6379', redisuid: '', mysqlport: '3306', mysqluid: '', mongodbport: '3717', mongodbuid: '',
+        clickhouseport: '9000', clickhouseuid: '', custom_ecs_info: '' },
       listLoading: false,
       dialogFormVisible: false,
       query: { vendor: '', account: '' },

vue-consul/src/views/node-exporter/jobs.vue

@@ -146,6 +146,7 @@
             <el-checkbox label="redis">REDIS</el-checkbox>
             <el-checkbox label="polardb">POLARDB</el-checkbox>
             <el-checkbox label="mongodb">MONGODB</el-checkbox>
+            <el-checkbox label="clickhouse">CLICKHOUSE</el-checkbox>
           </el-checkbox-group>
         </el-form-item>
         <el-form-item v-if="restype.includes('ecs')" label="优先获取外网IP" prop="isextip">
@@ -178,6 +179,9 @@
         <el-form-item v-if="restype.includes('mongodb')" label="MONGODB同步间隔(分钟)" prop="mongodb_interval">
           <el-input v-model="ecsJob.mongodb_interval" />
         </el-form-item>
+        <el-form-item v-if="restype.includes('clickhouse')" label="CLICKHOUSE同步间隔(分钟)" prop="clickhouse_interval">
+          <el-input v-model="ecsJob.clickhouse_interval" />
+        </el-form-item>
       </el-form>
 
       <div slot="footer" class="dialog-footer">
@@ -233,6 +237,7 @@
             <el-checkbox label="redis">REDIS</el-checkbox>
             <el-checkbox label="polardb">POLARDB</el-checkbox>
             <el-checkbox label="mongodb">MONGODB</el-checkbox>
+            <el-checkbox label="clickhouse">CLICKHOUSE</el-checkbox>
           </el-checkbox-group>
         </el-form-item>
         <el-form-item v-if="editJob.restype.includes('ecs')" label="优先获取外网IP" prop="isextip">
@@ -265,6 +270,9 @@
         <el-form-item v-if="editJob.restype.includes('mongodb')" label="MONGODB同步间隔(分钟)" prop="mongodb_interval">
           <el-input v-model="editJob.mongodb_interval" />
         </el-form-item>
+        <el-form-item v-if="editJob.restype.includes('clickhouse')" label="CLICKHOUSE同步间隔(分钟)" prop="clickhouse_interval">
+          <el-input v-model="editJob.clickhouse_interval" />
+        </el-form-item>
       </el-form>
 
       <div slot="footer" class="dialog-footer">
@@ -408,7 +416,7 @@ export default {
       },
 
       ecsJob: { vendor: '', ak: '', sk: '', region: [], account: '', proj_interval: 60, ecs_interval: 10,
-        rds_interval: 20, polardb_interval: 20, redis_interval: 20, mongodb_interval: 20 },
+        rds_interval: 20, polardb_interval: 20, redis_interval: 20, mongodb_interval: 20, clickhouse_interval: 20 },
       editJob: { restype: ['group'] },
       cloud_dict: {},
       upjob: { jobid: '', interval: '' },
@@ -442,6 +450,8 @@ export default {
         return 'warning-row'
       } else if (row.itype === 'mongodb') {
         return 'warning-row'
+      } else if (row.itype === 'clickhouse') {
+        return 'warning-row'
       }
       return ''
     },
@@ -471,18 +481,19 @@ export default {
         this.editJob.redis_interval = response.interval.redis_interval
         this.editJob.polardb_interval = response.interval.polardb_interval
         this.editJob.mongodb_interval = response.interval.mongodb_interval
+        this.editJob.clickhouse_interval = response.interval.clickhouse_interval
         this.listLoading = false
       })
     },
     handleEdit() {
-      this.editJob = { vendor: '', akskswitch: false, ak: '', sk: '', region: '', account: '', restype: ['group'], proj_interval: 60, ecs_interval: 10, rds_interval: 20, redis_interval: 20, polardb_interval: 20, mongodb_interval: 20, isextip: false }
+      this.editJob = { vendor: '', akskswitch: false, ak: '', sk: '', region: '', account: '', restype: ['group'], proj_interval: 60, ecs_interval: 10, rds_interval: 20, redis_interval: 20, polardb_interval: 20, mongodb_interval: 20, clickhouse_interval: 20, isextip: false }
       getCloud().then(response => {
         this.cloud_dict = response.cloud_dict
       })
       this.editFormVisible = true
     },
     handleCreate() {
-      this.ecsJob = { vendor: '', ak: '', sk: '', region: [], account: '', proj_interval: 60, ecs_interval: 10, rds_interval: 20, redis_interval: 20, polardb_interval: 20, mongodb_interval: 20, isextip: false }
+      this.ecsJob = { vendor: '', ak: '', sk: '', region: [], account: '', proj_interval: 60, ecs_interval: 10, rds_interval: 20, redis_interval: 20, polardb_interval: 20, mongodb_interval: 20, clickhouse_interval: 20, isextip: false }
       this.ecsJob.account = this.query.account
       this.newFormVisible = true
     },