From d3e5aa065a547e84169213176e29ee560b0ca7e7 Mon Sep 17 00:00:00 2001 From: zhangchunlin Date: Sun, 26 Jan 2025 18:09:16 +0800 Subject: [PATCH] add clickhouse jms --- flask-consul/requirements.txt | 2 + flask-consul/units/cloud/alicloud.py | 126 ++++++++++++++++++- flask-consul/units/cloud/sync_clickhouse.py | 77 ++++++++++++ flask-consul/units/jms/sync_jms.py | 17 ++- flask-consul/units/selfclickhouse_manager.py | 77 ++++++++++++ flask-consul/views/clickhouse.py | 93 ++++++++++++++ flask-consul/views/edit_cloud.py | 29 ++++- flask-consul/views/jms.py | 10 +- flask-consul/views/selfclickhouse.py | 80 ++++++++++++ vue-consul/src/views/jms/index.vue | 9 +- vue-consul/src/views/node-exporter/jobs.vue | 17 ++- 11 files changed, 522 insertions(+), 15 deletions(-) create mode 100644 flask-consul/units/cloud/sync_clickhouse.py create mode 100644 flask-consul/units/selfclickhouse_manager.py create mode 100644 flask-consul/views/clickhouse.py create mode 100644 flask-consul/views/selfclickhouse.py diff --git a/flask-consul/requirements.txt b/flask-consul/requirements.txt index 8d5cbe5..3ce2616 100644 --- a/flask-consul/requirements.txt +++ b/flask-consul/requirements.txt @@ -27,6 +27,8 @@ alibabacloud_bssopenapi20171214==2.0.6 aliyun-python-sdk-cms==7.0.32 alibabacloud_dds20151201==8.1.1 alibabacloud_polardb20170801==5.4.0 +alibabacloud_clickhouse20191111==3.1.12 +alibabacloud_clickhouse20230522==1.0.2 tencentcloud-sdk-python-common==3.0.779 tencentcloud-sdk-python-cvm==3.0.779 tencentcloud-sdk-python-cdb==3.0.779 diff --git a/flask-consul/units/cloud/alicloud.py b/flask-consul/units/cloud/alicloud.py index 7ebc245..c14daf4 100644 --- a/flask-consul/units/cloud/alicloud.py +++ b/flask-consul/units/cloud/alicloud.py @@ -17,10 +17,14 @@ from alibabacloud_polardb20170801.client import Client as polardb20170801Client from alibabacloud_polardb20170801 import models as polardb_20170801_models from alibabacloud_dds20151201.client import Client as Dds20151201Client from alibabacloud_dds20151201 import models as dds_20151201_models +from alibabacloud_clickhouse20191111.client import Client as clickhouse20191111Client +from alibabacloud_clickhouse20191111 import models as clickhouse_20191111_models +from alibabacloud_clickhouse20230522.client import Client as clickhouse20230522Client +from alibabacloud_clickhouse20230522 import models as clickhouse_20230522_models import sys,datetime,hashlib,math,traceback from units import consul_kv,consul_svc -from units.cloud import sync_ecs,sync_rds,sync_redis,notify,sync_polardb,sync_mongodb +from units.cloud import sync_ecs,sync_rds,sync_redis,notify,sync_polardb,sync_mongodb,sync_clickhouse from units.config_log import * def exp(account,collect_days,notify_days,notify_amount): @@ -396,7 +400,6 @@ def polardb(account, region): else: page_number += 1 try: - print("-------------1", polardb_dict) for iid in polardb_dict.keys(): logger.info(f'【ali_PolarDB】===> {iid}') describe_dbcluster_attribute_request = polardb_20170801_models.DescribeDBClusterEndpointsRequest( @@ -535,4 +538,121 @@ def mongodb(account, region): except Exception as e: logger.error(f'{e}\n{traceback.format_exc()}') data = {'count': '无', 'update': '失败', 'status': 50000, 'msg': str(e)} - consul_kv.put_kv(f'ConsulManager/record/jobs/alicloud/{account}/mongodb/{region}', data) \ No newline at end of file + consul_kv.put_kv(f'ConsulManager/record/jobs/alicloud/{account}/mongodb/{region}', data) + + +def clickhouse(account, region): + ak, sk = consul_kv.get_aksk('alicloud', account) + now = datetime.datetime.now().strftime('%m.%d/%H:%M') + group_dict = consul_kv.get_value(f'ConsulManager/assets/alicloud/group/{account}') + + config = open_api_models.Config(access_key_id=ak, access_key_secret=sk) + config.endpoint = 'clickhouse.aliyuncs.com' + if region == "ap-southeast-1": + config.endpoint = f'clickhouse.{region}.aliyuncs.com' + elif region == "eu-central-1": + config.endpoint = f'clickhouse.{region}.aliyuncs.com' + elif region == "us-west-1": + config.endpoint = f'clickhouse.{region}.aliyuncs.com' + client = clickhouse20191111Client(config) + page_number = 1 + clickhouse_dict = {} + runtime = util_models.RuntimeOptions() + try: + while True: + describe_dbinstances_request = clickhouse_20191111_models.DescribeDBClustersRequest( + page_size=100, + page_number=page_number, + region_id=region + ) + clickhouse_info = client.describe_dbclusters_with_options(describe_dbinstances_request, runtime) + clickhouse_list = clickhouse_info.body.to_map()['DBClusters']["DBCluster"] + + clickhouse_dict_temp = {i['DBClusterId']: { + 'name': i.get('DBClusterDescription', f"未命名{i['DBClusterId']}"), + 'domain': i['ConnectionString'], + 'ip': '无', + 'port': '9000', + 'region': region, + 'group': group_dict.get(i['ResourceGroupId'], '无'), + 'status': i['DBClusterStatus'], + 'itype': '社区版', + 'ver': i['DbVersion'], + 'exp': '-' if i.get('ExpireTime', None) is None else i.get('ExpireTime', '-T').split('T')[0], + 'cpu': '无', 'mem': '无', 'disk': '无' + } for i in clickhouse_list} + clickhouse_dict.update(clickhouse_dict_temp) + if len(clickhouse_list) < 100: + break + else: + page_number += 1 + + client = clickhouse20230522Client(config) + page_number = 1 + while True: + describe_dbinstances_request = clickhouse_20230522_models.DescribeDBInstancesRequest( + page_size=100, + page_number=page_number, + region_id=region + ) + clickhouse_info = client.describe_dbinstances_with_options(describe_dbinstances_request, runtime) + clickhouse_list = clickhouse_info.body.to_map()['Data']["DBInstances"] + + clickhouse_dict_temp = {i['DBInstanceId']: { + 'name': i.get('Description', f"未命名{i['DBInstanceId']}"), + 'domain': '无', + 'ip': '无', + 'port': '9000', + 'region': region, + 'group': group_dict.get(i['ResourceGroupId'], '无'), + 'status': i['Status'], + 'itype': '企业版', + 'ver': i['EngineVersion'], + 'exp': '-' if i.get('ExpireTime', None) is None else i.get('ExpireTime', '-T').split('T')[0], + 'cpu': '无', 'mem': '无', 'disk': '无' + } for i in clickhouse_list} + clickhouse_dict.update(clickhouse_dict_temp) + if len(clickhouse_list) < 100: + break + else: + page_number += 1 + try: + for iid in clickhouse_dict.keys(): + logger.info(f'【ali_Clickhouse】===> {iid}') + if clickhouse_dict[iid].get('domain') == '无': + describe_dbinstance_attribute_request = clickhouse_20230522_models.DescribeEndpointsRequest(region_id=region, dbinstance_id=iid) + clickhouse_plus_info = client.describe_endpoints_with_options(describe_dbinstance_attribute_request, runtime) + clickhouse_plus_list = clickhouse_plus_info.body.to_map()['Data']["Endpoints"] + clickhouse_plus= {} + for i in clickhouse_plus_list: + if i['NetType'] == 'VPC': + clickhouse_plus[iid] = { + 'domain': i['VpcInstanceId'], + 'ip': i['IPAddress'], + } + break + for k, v in clickhouse_plus.items(): + if k in clickhouse_dict: + clickhouse_dict[k].update(v) + except Exception as e: + logger.error('DescribeDBInstancesAsCsvRequest ERROR' + f'{e}\n{traceback.format_exc()}') + count = len(clickhouse_dict) + off, on = sync_clickhouse.w2consul('alicloud', account, region, clickhouse_dict) + data = {'count': count, 'update': now, 'status': 20000, 'on': on, 'off': off, + 'msg': f'clickhouse同步成功！总数：{count}，开机：{on}，关机：{off}'} + consul_kv.put_kv(f'ConsulManager/record/jobs/alicloud/{account}/clickhouse/{region}', data) + logger.info(f'【JOB】===>alicloud_clickhouse {account} {region} {data}') + except TeaException as e: + emsg = e.message.split('. ', 1)[0] + logger.error(f"【code:】{e.code}\n【message:】{e.message}\n{traceback.format_exc()}") + data = consul_kv.get_value(f'ConsulManager/record/jobs/alicloud/{account}/clickhouse/{region}') + if data == {}: + data = {'count': '无', 'update': f'失败{e.code}', 'status': 50000, 'msg': emsg} + else: + data['update'] = f'失败{e.code}' + data['msg'] = emsg + consul_kv.put_kv(f'ConsulManager/record/jobs/alicloud/{account}/clickhouse/{region}', data) + except Exception as e: + logger.error(f'{e}\n{traceback.format_exc()}') + data = {'count': '无', 'update': '失败', 'status': 50000, 'msg': str(e)} + consul_kv.put_kv(f'ConsulManager/record/jobs/alicloud/{account}/clickhouse/{region}', data) \ No newline at end of file diff --git a/flask-consul/units/cloud/sync_clickhouse.py b/flask-consul/units/cloud/sync_clickhouse.py new file mode 100644 index 0000000..7b438f3 --- /dev/null +++ b/flask-consul/units/cloud/sync_clickhouse.py @@ -0,0 +1,77 @@ +#!/usr/bin/python3 +import requests,json +from units import consul_kv +from config import consul_token,consul_url,vendors,regions +from units.config_log import * +headers = {'X-Consul-Token': consul_token} +geturl = f'{consul_url}/agent/services' +delurl = f'{consul_url}/agent/service/deregister' +puturl = f'{consul_url}/agent/service/register' +def w2consul(vendor,account,region,clickhouse_dict): + service_name = f'{vendor}_{account}_clickhouse' + params = {'filter': f'Service == "{service_name}" and "{region}" in Tags and Meta.account == "{account}"'} + try: + consul_clickhouse_iid_list = requests.get(geturl, headers=headers, params=params).json().keys() + except: + consul_clickhouse_iid_list = [] + + #在consul中删除云厂商不存在的clickhouse + for del_clickhouse in [x for x in consul_clickhouse_iid_list if x not in clickhouse_dict.keys()]: + dereg = requests.put(f'{delurl}/{del_clickhouse}', headers=headers) + if dereg.status_code == 200: + logger.info(f"code: 20000, data: {account}-删除成功！") + else: + logger.info(f"code: 50000, data: {dereg.status_code}:{dereg.text}") + off,on = 0,0 + for k,v in clickhouse_dict.items(): + iid = k + #对consul中关机的clickhouse做标记。 + if v['status'] in ['SHUTDOWN','非运行中']: + off = off + 1 + tags = ['OFF',v['itype'],v['ver'], region] + stat = 'off' + else: + on = on + 1 + tags = ['ON',v['itype'],v['ver'],region] + stat = 'on' + custom_clickhouse = consul_kv.get_value(f'ConsulManager/assets/sync_clickhouse_custom/{iid}') + port = custom_clickhouse.get('port') + ip = custom_clickhouse.get('ip') + if port == None: + port = v['port'] + if ip == None: + ip = v['ip'] + instance = f'{ip}:{port}' + data = { + 'id': iid, + 'name': service_name, + 'Address': ip, + 'port': port, + 'tags': tags, + 'Meta': { + 'iid': iid, + 'name': v['name'], + 'region': regions[vendor].get(region,'未找到'), + 'group': v['group'], + 'instance': instance, + 'account': account, + 'itype': v['itype'], + 'vendor': vendors.get(vendor,'未找到'), + 'os': "clickhouse", + 'ver': v['ver'], + 'domain':v['domain'], + 'exp': v['exp'], + 'stat': stat, + 'team': v.get('team','无') + }, + "check": { + "tcp": f"{ip}:{port}", + "interval": "60s" + } + } + reg = requests.put(puturl, headers=headers, data=json.dumps(data)) + if reg.status_code == 200: + pass + else: + logger.info(f"{account}:code: 5000, data: {reg.status_code}:{reg.text}") + return off,on diff --git a/flask-consul/units/jms/sync_jms.py b/flask-consul/units/jms/sync_jms.py index 878c5b2..1932db2 100755 --- a/flask-consul/units/jms/sync_jms.py +++ b/flask-consul/units/jms/sync_jms.py @@ -2,7 +2,7 @@ import datetime,requests,json,traceback from units import consul_kv,consul_manager,myaes from units.config_log import * -resource_type = ["ecs", "redis", "rds", "mongodb", "polardb"] +resource_type = ["ecs", "redis", "rds", "mongodb", "polardb", "clickhouse"] # 创建节点 @@ -110,19 +110,24 @@ def update_jms(jms_ver,jms_url,headers,new_node_dict,node_id,cloud,account,ecs_i ecs_url = f"{jms_url}/api/v1/assets/databases/" payload["protocols"] = [{"name": proto, "port": port}] payload.update({"db_name": "0", "use_ssl": False, "allow_invalid_cert": False}) - if float(v['ver']) < 6: - payload["platform"] = '24' - else: - payload["platform"] = "25" + # if float(v['ver']) < 6: + # payload["platform"] = '24' + # else: + payload["platform"] = "25" elif platform == 'Mysql': ecs_url = f"{jms_url}/api/v1/assets/databases/" payload["platform"] = '17' payload["protocols"] = [{"name": proto, "port": port}] elif platform == 'Mongodb': ecs_url = f"{jms_url}/api/v1/assets/databases/" - payload["platform"] = '23' + payload["platform"] = '33' payload["protocols"] = [{"name": proto, "port": port}] payload.update({"db_name": "admin"}) + elif platform == 'Clickhouse': + ecs_url = f"{jms_url}/api/v1/assets/databases/" + payload["platform"] = '22' + payload["protocols"] = [{"name": proto, "port": port}] + payload.update({"db_name": "default"}) else: logger.error(f"未匹配到{platform}！") continue diff --git a/flask-consul/units/selfclickhouse_manager.py b/flask-consul/units/selfclickhouse_manager.py new file mode 100644 index 0000000..026e43a --- /dev/null +++ b/flask-consul/units/selfclickhouse_manager.py @@ -0,0 +1,77 @@ +import requests,json,re +from config import consul_token,consul_url +from units.config_log import * +headers = {'X-Consul-Token': consul_token} + +def get_all_list(vendor,account,region,group): + vendor = f'and Meta.vendor=="{vendor}"' if vendor != '' else f'and Meta.vendor != ""' + account = f'and Meta.account=="{account}"' if account != '' else f'and Meta.account != ""' + region = f'and Meta.region=="{region}"' if region != '' else f'and Meta.region != ""' + group = f'and Meta.group=="{group}"' if group != '' else f'and Meta.group != ""' + url = f'{consul_url}/agent/services?filter=Service == selfrds_exporter {vendor} {account} {region} {group}' + response = requests.get(url, headers=headers) + if response.status_code == 200: + info = response.json() + all_list = [i['Meta'] for i in info.values()] + vendor_list = sorted(list(set([i['vendor'] for i in all_list]))) + account_list = sorted(list(set([i['account'] for i in all_list]))) + region_list = sorted(list(set([i['region'] for i in all_list]))) + group_list = sorted(list(set([i['group'] for i in all_list]))) + return {'code': 20000,'all_list':all_list,'vendor_list':vendor_list, + 'account_list':account_list,'region_list':region_list,'group_list':group_list} + else: + logger.error(f"{response.status_code}：{response.text}") + return {'code': 50000, 'data': f'{response.status_code}:{response.text}'} + +def get_service(): + response = requests.get(f'{consul_url}/agent/services?filter=Service == selfrds_exporter', headers=headers) + if response.status_code == 200: + info = response.json() + all_list = [i['Meta'] for i in info.values()] + vendor_list = sorted(list(set([i['vendor'] for i in all_list]))) + account_list = sorted(list(set([i['account'] for i in all_list]))) + region_list = sorted(list(set([i['region'] for i in all_list]))) + group_list = sorted(list(set([i['group'] for i in all_list]))) + return {'code': 20000,'all_list':all_list,'vendor_list':vendor_list, + 'account_list':account_list,'region_list':region_list,'group_list':group_list} + else: + logger.error(f"{response.status_code}：{response.text}") + return {'code': 50000, 'data': f'{response.status_code}:{response.text}'} + +def add_service(vendor,account,region,group,name,ip,port,os): + if port is None or name is None: + return {"code": 50000, "data": f"名称或IP不能为空！"} + sid = f"{vendor}/{account}/{region}/{group}@{name}".strip() + #sid = eval(repr(sid).replace('\\t','').replace('\\n','')) + sid = re.sub('[[ \]`~!\\\#$^&*=|"{}\':;?\t\n]','_',sid)+'@rds' + instance = f'{ip}:{port}' + if '//' in sid or sid.startswith('/'): + return {"code": 50000, "data": f"服务ID【{sid}】首尾不能包含'/'，并且不能包含两个连续的'/'"} + data = { + "id": sid, + "name": 'selfrds_exporter', + 'Address': ip, + 'port': int(port), + "tags": [vendor,os], + "Meta": {'vendor':vendor,'account':account,'region':region,'group':group, + 'name':name,'instance':instance,'os':os}, + "check": {"tcp": instance,"interval": "60s"} + } + reg = requests.put(f'{consul_url}/agent/service/register', headers=headers, data=json.dumps(data)) + if reg.status_code == 200: + return {"code": 20000, "data": f"【{sid}】增加成功！"} + else: + logger.error(f"{reg.status_code}【{sid}】{reg.text}") + return {"code": 50000, "data": f"{reg.status_code}【{sid}】{reg.text}"} + +def del_service(vendor,account,region,group,name): + sid = f"{vendor}/{account}/{region}/{group}@{name}".strip() + sid = re.sub('[[ \]`~!\\\#$^&*=|"{}\':;?\t\n]','_',sid)+'@rds' + reg = requests.put(f'{consul_url}/agent/service/deregister/{sid}', headers=headers) + if reg.status_code == 200: + logger.debug(f"【{sid}】删除成功！") + return {"code": 20000, "data": f"【{sid}】删除成功！"} + else: + logger.error(f"{reg.status_code}【{sid}】{reg.text}") + return {"code": 50000, "data": f"{reg.status_code}【{sid}】{reg.text}"} + diff --git a/flask-consul/views/clickhouse.py b/flask-consul/views/clickhouse.py new file mode 100644 index 0000000..20500a0 --- /dev/null +++ b/flask-consul/views/clickhouse.py @@ -0,0 +1,93 @@ +from flask import Blueprint +from flask_restful import reqparse, Resource, Api +from flask_apscheduler import APScheduler +import traceback +#import sys +#sys.path.append("..") +from units import token_auth,consul_kv,gen_config,consul_svc +from units.config_log import * +blueprint = Blueprint('clickhouse',__name__) +api = Api(blueprint) + +parser = reqparse.RequestParser() +parser.add_argument('job_id',type=str) +parser.add_argument('services_dict',type=dict) +parser.add_argument('cst_clickhouse_dict',type=dict) +parser.add_argument('iid',type=str) +parser.add_argument('jobclickhouse_name',type=str) +parser.add_argument('checked',type=str) + +class Clickhouse(Resource): + decorators = [token_auth.auth.login_required] + def get(self, stype): + job_id = parser.parse_args()['job_id'] + if stype == 'jobclickhouse': + jobclickhouse = consul_kv.get_keys_list('ConsulManager/jobs') + jobclickhouse_list = [i.split('/jobs/')[1] for i in jobclickhouse if '/clickhouse/' in i] + return {'code': 20000,'jobclickhouse':jobclickhouse_list} + elif stype == 'clickhouse_services': + jobecs = consul_kv.get_keys_list('ConsulManager/jobs') + jobecs_list = [i.split('/jobs/')[1] for i in jobecs if '/clickhouse/' in i] + services_list = [] + for i in jobecs_list: + serivces = i.split("/") + services_list.append(f'{serivces[0]}_{serivces[1]}_{serivces[2]}') + return {'code': 20000,'services_list': sorted(set(services_list))} + elif stype == 'clickhouserules': + return gen_config.get_clickhouserules() + elif stype == 'cstclickhouseconf': + args = parser.parse_args() + iid = args['iid'] + cst_clickhouse_config = consul_kv.get_value(f'ConsulManager/assets/sync_clickhouse_custom/{iid}') + cst_clickhouse_config.update({'iid': iid,'ipswitch': False,'portswitch': False}) + if 'ip' in cst_clickhouse_config and cst_clickhouse_config['ip'] != '': + cst_clickhouse_config['ipswitch'] = True + if 'port' in cst_clickhouse_config and cst_clickhouse_config['port'] != '': + cst_clickhouse_config['portswitch'] = True + return {'code': 20000, 'cst_clickhouse': cst_clickhouse_config} + elif stype == 'cstclickhouselist': + args = parser.parse_args() + jobclickhouse_name = args['jobclickhouse_name'] + checked = args['checked'] + cst_clickhouse_dict = consul_kv.get_kv_dict('ConsulManager/assets/sync_clickhouse_custom/') + cst_clickhouse_keylist = [k.split('/')[-1] for k,v in cst_clickhouse_dict.items() if v != {}] + clickhouse_info = consul_kv.get_res_services(jobclickhouse_name) + if checked == 'false': + return clickhouse_info + else: + cst_clickhouse_list = [i for i in clickhouse_info['res_list'] if i['iid'] in cst_clickhouse_keylist] + return {'code': 20000, 'res_list': cst_clickhouse_list} + + def post(self, stype): + if stype == 'clickhousepconfig': + args = parser.parse_args() + services_dict = args['services_dict'] + return gen_config.clickhouse_config(services_dict['jobclickhouse_list'],services_dict['cm_exporter'],services_dict['services_list'],services_dict['exporter']) + elif stype == 'cstclickhouse': + args = parser.parse_args() + cst_clickhouse_dict = args['cst_clickhouse_dict'] + consul_clickhouse_cst = {} + iid = cst_clickhouse_dict['iid'] + try: + sid_dict = consul_svc.get_sid(iid)['instance'] + if cst_clickhouse_dict['portswitch'] and cst_clickhouse_dict['port'] != '': + consul_clickhouse_cst['port'] = int(cst_clickhouse_dict['port']) + sid_dict['Port'] = consul_clickhouse_cst['port'] + if cst_clickhouse_dict['ipswitch'] and cst_clickhouse_dict['ip'] != '': + consul_clickhouse_cst['ip'] = cst_clickhouse_dict['ip'] + sid_dict['Address'] = consul_clickhouse_cst['ip'] + consul_kv.put_kv(f'ConsulManager/assets/sync_clickhouse_custom/{iid}',consul_clickhouse_cst) + del sid_dict['Weights'] + del sid_dict['ContentHash'] + del sid_dict['Datacenter'] + sid_dict['name'] = sid_dict.pop('Service') + sid_dict['Meta']['instance'] = f"{sid_dict['Address']}:{sid_dict['Port']}" + sid_dict["check"] = { "tcp": sid_dict['Meta']['instance'],"interval": "60s" } + consul_svc.del_sid(iid) + consul_svc.add_sid(sid_dict) + return {'code': 20000, 'data': '自定义实例信息修改成功！'} + except Exception as e: + logger.error(f'{e}\n{traceback.format_exc()}') + return {'code': 50000, "data": '提交自定义实例信息格式错误！'} + +api.add_resource(Clickhouse, '/api/clickhouse/') diff --git a/flask-consul/views/edit_cloud.py b/flask-consul/views/edit_cloud.py index 6eeffd3..5f0e529 100644 --- a/flask-consul/views/edit_cloud.py +++ b/flask-consul/views/edit_cloud.py @@ -35,7 +35,7 @@ class Edit(Resource): region = args['region'] restype = ['group'] interval = {'proj_interval': 60, 'ecs_interval': 10, 'rds_interval': 20, 'redis_interval': 20, - 'polardb_interval': 20, 'mongodb_interval': 20} + 'polardb_interval': 20, 'mongodb_interval': 20, 'clickhouse_interval': 20} isextip = False for i in job_list: if f'{vendor}/{account}/group' == i['id']: @@ -56,6 +56,9 @@ class Edit(Resource): elif f'{vendor}/{account}/mongodb/{region}' == i['id']: restype.append('mongodb') interval['mongodb_interval'] = i['minutes'] + elif f'{vendor}/{account}/clickhouse/{region}' == i['id']: + restype.append('clickhouse') + interval['clickhouse_interval'] = i['minutes'] return {'code': 20000, 'restype': restype, 'interval': interval, 'isextip': isextip} def post(self,stype): job_list = list(consul_kv.get_kv_dict(f'ConsulManager/jobs').values()) @@ -73,6 +76,7 @@ class Edit(Resource): polardb_interval = int(editjob_dict['polardb_interval']) mongodb_interval = int(editjob_dict['mongodb_interval']) redis_interval = int(editjob_dict['redis_interval']) + clickhouse_interval = int(editjob_dict['clickhouse_interval']) logger.info(f'{editjob_dict}') if editjob_dict['akskswitch']: ak = editjob_dict['ak'] @@ -91,6 +95,7 @@ class Edit(Resource): polardb_jobid = f'{vendor}/{account}/polardb/{region}' mongodb_jobid = f'{vendor}/{account}/mongodb/{region}' redis_jobid = f'{vendor}/{account}/redis/{region}' + clickhouse_jobid = f'{vendor}/{account}/clickhouse/{region}' if 'ecs' in restype: isecs = [x for x in job_list if x['id'] == f'{vendor}/{account}/ecs/{region}'] if len(isecs) == 1: @@ -209,5 +214,27 @@ class Edit(Resource): except: pass + if 'clickhouse' in restype: + isclickhouse = [x for x in job_list if x['id'] == f'{vendor}/{account}/clickhouse/{region}'] + if len(isclickhouse) == 1: + if clickhouse_interval != isclickhouse[0]['minutes']: + isclickhouse[0]['minutes'] = clickhouse_interval + consul_kv.put_kv(f'ConsulManager/jobs/{clickhouse_jobid}',isclickhouse[0]) + modjob_interval(clickhouse_jobid,clickhouse_interval) + else: + job_func = f"__main__:{vendor}.clickhouse" + job_args = [account,region] + job_interval = clickhouse_interval + addjob(clickhouse_jobid, job_func, job_args, job_interval) + job_dict = {'id':clickhouse_jobid,'func':job_func,'args':job_args,'minutes':job_interval, + "trigger": "interval","replace_existing": True} + consul_kv.put_kv(f'ConsulManager/jobs/{clickhouse_jobid}',job_dict) + else: + try: + consul_kv.del_key(f'ConsulManager/jobs/{clickhouse_jobid}') + deljob(clickhouse_jobid) + except: + pass + return {'code': 20000, 'data': f'{vendor}/{account}/{region}：编辑成功！'} api.add_resource(Edit, '/api/edit/') diff --git a/flask-consul/views/jms.py b/flask-consul/views/jms.py index b49fb3e..3510e19 100644 --- a/flask-consul/views/jms.py +++ b/flask-consul/views/jms.py @@ -45,7 +45,7 @@ class Jms(Resource): services_meta = consul_kv.get_services_meta(f'{vendor}_{account}_ecs').get('ecs_list', []) count_ecs = len(services_meta) count_off, count_on, count_cpu, count_mem, count_win, count_linux = 0, 0, 0, 0, 0, 0 - count_redis, count_mongodb, count_mysql = 0, 0, 0 + count_redis, count_mongodb, count_mysql, count_clickhouse = 0, 0, 0, 0 for i in services_meta: if i['os'] == 'linux': count_linux = count_linux + 1 @@ -65,6 +65,8 @@ class Jms(Resource): count_mysql = count_mysql + 1 elif i['os'] == 'mongodb': count_mongodb = count_mongodb + 1 + elif i['os'] == 'clickhouse': + count_clickhouse = count_clickhouse + 1 if i.get('stat') == 'off': count_off = count_off + 1 else: @@ -89,6 +91,7 @@ class Jms(Resource): 'count_redis': count_redis, 'count_mysql': count_mysql, 'count_mongodb': count_mongodb, + 'count_clickhouse': count_clickhouse, 'count_mem': f'{count_mem}GB', 'count_cpu': f'{count_cpu}核', 'count_ecs': count_ecs, @@ -119,6 +122,8 @@ class Jms(Resource): mysqluid = ecs_info['mysql'][-1] mongodbport = ecs_info['mongodb'][0][0].split('/')[-1] mongodbuid = ecs_info['mongodb'][-1] + clickhouseport = ecs_info['clickhouse'][0][0].split('/')[-1] + clickhouseuid = ecs_info['clickhouse'][-1] token = myaes.decrypt(jms_info['token']) custom_ecs_json = json.dumps(custom_ecs_info, indent=8) if custom_ecs_info != {} else '' @@ -136,6 +141,8 @@ class Jms(Resource): 'mysqluid': mysqluid, 'mongodbport': mongodbport, 'mongodbuid': mongodbuid, + 'clickhouseport': clickhouseport, + 'clickhouseuid': clickhouseuid, 'custom_ecs_info': custom_ecs_json, } else: @@ -155,6 +162,7 @@ class Jms(Resource): "redis": [[f"redis/{jms_config['redisport']}"], jms_config['redisuid']], "mysql": [[f"mysql/{jms_config['mysqlport']}"], jms_config['mysqluid']], "mongodb": [[f"mongodb/{jms_config['mongodbport']}"], jms_config['mongodbuid']], + "clickhouse": [[f"clickhouse/{jms_config['clickhouseport']}"], jms_config['clickhouseuid']], } consul_kv.put_kv('ConsulManager/jms/ecs_info', ecs_info) custom_ecs_info = jms_config['custom_ecs_info'] diff --git a/flask-consul/views/selfclickhouse.py b/flask-consul/views/selfclickhouse.py new file mode 100644 index 0000000..91e74c0 --- /dev/null +++ b/flask-consul/views/selfclickhouse.py @@ -0,0 +1,80 @@ +import os +from flask import Blueprint +from flask_restful import reqparse, Resource, Api +import sys,traceback +sys.path.append("..") +from units import token_auth,selfclickhouse_manager +from werkzeug.datastructures import FileStorage +from units import upload +from units.config_log import * +blueprint = Blueprint('selfclickhouse',__name__) +api = Api(blueprint) + +parser = reqparse.RequestParser() +parser.add_argument('vendor',type=str) +parser.add_argument('account',type=str) +parser.add_argument('region',type=str) +parser.add_argument('group',type=str) +parser.add_argument('name',type=str) +parser.add_argument('ip',type=str) +parser.add_argument('port',type=str) +parser.add_argument('os',type=str) +parser.add_argument('del_dict',type=dict) +parser.add_argument('up_dict',type=dict) +parser.add_argument('file',type=FileStorage, location="files", help="File is wrong.") + +class Upload(Resource): + @token_auth.auth.login_required + def post(self): + file = parser.parse_args().get("file") + try: + filename = file.filename + file_extension = os.path.splitext(filename)[1].lower() + try: + file_data = file.read() + if file_extension == '.xlsx': + return upload.read_execl(file_data,'selfclickhouse') + elif file_extension == '.csv': + return upload.read_csv(file_data,'selfclickhouse') + except Exception as e: + logger.error(f"【selfclickhouse】文件后缀名错误，请导入xlsx或csv格式,{e}\n{traceback.format_exc()}") + return {"code": 50000, "data": f"文件后缀名错误，请导入xlsx或csv格式！"} + except Exception as e: + logger.error(f"【selfclickhouse】导入失败,{e}\n{traceback.format_exc()}") + return {"code": 50000, "data": f"导入失败！"} + +class GetAllList(Resource): + @token_auth.auth.login_required + def get(self): + args = parser.parse_args() + return selfclickhouse_manager.get_all_list(args['vendor'],args['account'],args['region'],args['group']) + +class SelfclickhouseApi(Resource): + decorators = [token_auth.auth.login_required] + def get(self): + return selfclickhouse_manager.get_service() + def post(self): + args = parser.parse_args() + logger.info(f'=======\n{args}') + return selfclickhouse_manager.add_service(args['vendor'],args['account'],args['region'], + args['group'],args['name'],args['ip'],args['port'],args['os']) + def put(self): + args = parser.parse_args() + del_dict = args['del_dict'] + up_dict = args['up_dict'] + resp_del = selfclickhouse_manager.del_service(del_dict['vendor'],del_dict['account'], + del_dict['region'],del_dict['group'],del_dict['name']) + resp_add = selfclickhouse_manager.add_service(up_dict['vendor'],up_dict['account'],up_dict['region'], + up_dict['group'],up_dict['name'],up_dict['ip'], + up_dict['port'],up_dict['os']) + if resp_del["code"] == 20000 and resp_add["code"] == 20000: + return {"code": 20000, "data": f"更新成功！"} + else: + return {"code": 50000, "data": f"更新失败！"} + def delete(self): + args = parser.parse_args() + return selfclickhouse_manager.del_service(args['vendor'],args['account'],args['region'],args['group'],args['name']) + +api.add_resource(GetAllList,'/api/selfclickhouse/alllist') +api.add_resource(SelfclickhouseApi, '/api/selfclickhouse/service') +api.add_resource(Upload,'/api/selfclickhouse/upload') diff --git a/vue-consul/src/views/jms/index.vue b/vue-consul/src/views/jms/index.vue index d337df3..46e243f 100644 --- a/vue-consul/src/views/jms/index.vue +++ b/vue-consul/src/views/jms/index.vue @@ -67,6 +67,11 @@ mongodb端口：   管理用户ID： +
+

Clickhouse：

+ clickhouse端口： +   管理用户ID： +

Mysql：

mysql端口： @@ -95,6 +100,7 @@ + @@ -149,7 +155,8 @@ export default { data() { return { jms_config: { ver: 'V2', url: '', token: '', linuxport: '22', linuxuid: '', winport: '3389', winuid: '', - redisport: '6379', redisuid: '', mysqlport: '3306', mysqluid: '', mongodbport: '3717', mongodbuid: '', custom_ecs_info: '' }, + redisport: '6379', redisuid: '', mysqlport: '3306', mysqluid: '', mongodbport: '3717', mongodbuid: '', + clickhouseport: '9000', clickhouseuid: '', custom_ecs_info: '' }, listLoading: false, dialogFormVisible: false, query: { vendor: '', account: '' }, diff --git a/vue-consul/src/views/node-exporter/jobs.vue b/vue-consul/src/views/node-exporter/jobs.vue index 7ee9fe8..d759f0e 100644 --- a/vue-consul/src/views/node-exporter/jobs.vue +++ b/vue-consul/src/views/node-exporter/jobs.vue @@ -146,6 +146,7 @@ REDIS POLARDB MONGODB + CLICKHOUSE @@ -178,6 +179,9 @@ + + +
@@ -233,6 +237,7 @@ REDIS POLARDB MONGODB + CLICKHOUSE @@ -265,6 +270,9 @@ + + +
@@ -408,7 +416,7 @@ export default { }, ecsJob: { vendor: '', ak: '', sk: '', region: [], account: '', proj_interval: 60, ecs_interval: 10, - rds_interval: 20, polardb_interval: 20, redis_interval: 20, mongodb_interval: 20 }, + rds_interval: 20, polardb_interval: 20, redis_interval: 20, mongodb_interval: 20, clickhouse_interval: 20 }, editJob: { restype: ['group'] }, cloud_dict: {}, upjob: { jobid: '', interval: '' }, @@ -442,6 +450,8 @@ export default { return 'warning-row' } else if (row.itype === 'mongodb') { return 'warning-row' + } else if (row.itype === 'clickhouse') { + return 'warning-row' } return '' }, @@ -471,18 +481,19 @@ export default { this.editJob.redis_interval = response.interval.redis_interval this.editJob.polardb_interval = response.interval.polardb_interval this.editJob.mongodb_interval = response.interval.mongodb_interval + this.editJob.clickhouse_interval = response.interval.clickhouse_interval this.listLoading = false }) }, handleEdit() { - this.editJob = { vendor: '', akskswitch: false, ak: '', sk: '', region: '', account: '', restype: ['group'], proj_interval: 60, ecs_interval: 10, rds_interval: 20, redis_interval: 20, polardb_interval: 20, mongodb_interval: 20, isextip: false } + this.editJob = { vendor: '', akskswitch: false, ak: '', sk: '', region: '', account: '', restype: ['group'], proj_interval: 60, ecs_interval: 10, rds_interval: 20, redis_interval: 20, polardb_interval: 20, mongodb_interval: 20, clickhouse_interval: 20, isextip: false } getCloud().then(response => { this.cloud_dict = response.cloud_dict }) this.editFormVisible = true }, handleCreate() { - this.ecsJob = { vendor: '', ak: '', sk: '', region: [], account: '', proj_interval: 60, ecs_interval: 10, rds_interval: 20, redis_interval: 20, polardb_interval: 20, mongodb_interval: 20, isextip: false } + this.ecsJob = { vendor: '', ak: '', sk: '', region: [], account: '', proj_interval: 60, ecs_interval: 10, rds_interval: 20, redis_interval: 20, polardb_interval: 20, mongodb_interval: 20, clickhouse_interval: 20, isextip: false } this.ecsJob.account = this.query.account this.newFormVisible = true },