diff --git a/flask-consul/requirements.txt b/flask-consul/requirements.txt index 07c5e3e..3ce2616 100644 --- a/flask-consul/requirements.txt +++ b/flask-consul/requirements.txt @@ -25,6 +25,10 @@ alibabacloud_rds20140815==2.1.2 alibabacloud_r_kvstore20150101==2.20.7 alibabacloud_bssopenapi20171214==2.0.6 aliyun-python-sdk-cms==7.0.32 +alibabacloud_dds20151201==8.1.1 +alibabacloud_polardb20170801==5.4.0 +alibabacloud_clickhouse20191111==3.1.12 +alibabacloud_clickhouse20230522==1.0.2 tencentcloud-sdk-python-common==3.0.779 tencentcloud-sdk-python-cvm==3.0.779 tencentcloud-sdk-python-cdb==3.0.779 diff --git a/flask-consul/units/cloud/alicloud.py b/flask-consul/units/cloud/alicloud.py index 2a0c62e..745472d 100644 --- a/flask-consul/units/cloud/alicloud.py +++ b/flask-consul/units/cloud/alicloud.py @@ -13,10 +13,18 @@ from alibabacloud_rds20140815.client import Client as Rds20140815Client from alibabacloud_rds20140815 import models as rds_20140815_models from alibabacloud_r_kvstore20150101 import models as r_kvstore_20150101_models from alibabacloud_r_kvstore20150101.client import Client as R_kvstore20150101Client +from alibabacloud_polardb20170801.client import Client as polardb20170801Client +from alibabacloud_polardb20170801 import models as polardb_20170801_models +from alibabacloud_dds20151201.client import Client as Dds20151201Client +from alibabacloud_dds20151201 import models as dds_20151201_models +from alibabacloud_clickhouse20191111.client import Client as clickhouse20191111Client +from alibabacloud_clickhouse20191111 import models as clickhouse_20191111_models +from alibabacloud_clickhouse20230522.client import Client as clickhouse20230522Client +from alibabacloud_clickhouse20230522 import models as clickhouse_20230522_models import sys,datetime,hashlib,math,traceback from units import consul_kv,consul_svc -from units.cloud import sync_ecs,sync_rds,sync_redis,notify +from units.cloud import sync_ecs,sync_rds,sync_redis,notify,sync_polardb,sync_mongodb,sync_clickhouse from units.config_log import * def exp(account,collect_days,notify_days,notify_amount): @@ -193,6 +201,12 @@ def redis(account,region): config = open_api_models.Config(access_key_id=ak,access_key_secret=sk) config.endpoint = 'r-kvstore.aliyuncs.com' + if region == "ap-southeast-1": + config.endpoint = f'r-kvstore.{region}.aliyuncs.com' + elif region == "eu-central-1": + config.endpoint = f'r-kvstore.{region}.aliyuncs.com' + elif region == "us-west-1": + config.endpoint = f'r-kvstore.{region}.aliyuncs.com' client = R_kvstore20150101Client(config) PageNumber = 1 nextpage = True @@ -255,6 +269,12 @@ def rds(account,region): config = open_api_models.Config(access_key_id=ak,access_key_secret=sk) config.endpoint = 'rds.aliyuncs.com' + if region == "ap-southeast-1": + config.endpoint = f'rds.{region}.aliyuncs.com' + elif region == "eu-central-1": + config.endpoint = f'rds.{region}.aliyuncs.com' + elif region == "us-west-1": + config.endpoint = f'rds.{region}.aliyuncs.com' client = Rds20140815Client(config) next_token = '0' @@ -329,3 +349,310 @@ def rds(account,region): logger.error(f'{e}\n{traceback.format_exc()}') data = {'count':'无','update':f'失败','status':50000,'msg':str(e)} consul_kv.put_kv(f'ConsulManager/record/jobs/alicloud/{account}/rds/{region}', data) + + +def polardb(account, region): + ak, sk = consul_kv.get_aksk('alicloud', account) + now = datetime.datetime.now().strftime('%m.%d/%H:%M') + group_dict = consul_kv.get_value(f'ConsulManager/assets/alicloud/group/{account}') + + config = open_api_models.Config(access_key_id=ak, access_key_secret=sk) + config.endpoint = 'polardb.aliyuncs.com' + if region == "ap-southeast-1": + config.endpoint = f'polardb.{region}.aliyuncs.com' + elif region == "eu-central-1": + config.endpoint = f'polardb.{region}.aliyuncs.com' + elif region == "us-west-1": + config.endpoint = f'polardb.{region}.aliyuncs.com' + client = polardb20170801Client(config) # 使用PolarDB客户端 + + page_number = 1 + polardb_dict = {} + runtime = util_models.RuntimeOptions() + try: + while True: + describe_dbclusters_request = polardb_20170801_models.DescribeDBClustersRequest( + page_size=100, + dbtype="MySQL", + region_id=region, + page_number=page_number, + ) + + cluster_info = client.describe_dbclusters_with_options(describe_dbclusters_request, runtime) + cluster_list = cluster_info.body.to_map()['Items']['DBCluster'] + + polardb_dict_temp = {i['DBClusterId']: { + 'name': i.get('DBClusterDescription', f"未命名{i['DBClusterId']}"), + 'domain': '无', # todo + 'ip': '无', # todo + 'port': 3306, # todo + 'region': region, + 'group': group_dict.get(i['ResourceGroupId'], '无'), + 'status': i['DBClusterStatus'], + 'itype': '无', # todo + 'ver': i['DBVersion'], + 'exp': '-' if i.get('ExpireTime', None) == None else i.get('ExpireTime', '-T').split('T')[0], + 'cpu': '无', 'mem': '无', 'disk': '无' + } for i in cluster_list} + polardb_dict.update(polardb_dict_temp) + if len(cluster_list) < 100: + break + else: + page_number += 1 + try: + for iid in polardb_dict.keys(): + logger.info(f'【ali_PolarDB】===> {iid}') + describe_dbcluster_attribute_request = polardb_20170801_models.DescribeDBClusterEndpointsRequest( + dbcluster_id=iid) + cluster_plus_info = client.describe_dbcluster_endpoints_with_options( + describe_dbcluster_attribute_request, runtime) + cluster_plus_list = cluster_plus_info.body.to_map()['Items'] + polardb_plus = {} + for i in cluster_plus_list: + if i['EndpointType'] == 'Cluster': + for address_item in i['AddressItems']: + if address_item['NetType'] == 'Private': + # 获取 DBClusterId 作为键 + db_cluster_id = iid + # 构建子字典并将其赋值给 polardb_plus + polardb_plus[db_cluster_id] = { + 'port': int(address_item['Port']), + 'domain': address_item['ConnectionString'], + 'ip': address_item['IPAddress'], + 'itype': i['EndpointType'], + } + break + for k, v in polardb_plus.items(): + if k in polardb_dict: + polardb_dict[k].update(v) + except Exception as e: + logger.error('DescribeDBClustersAsCsvRequest ERROR' + f'{e}\n{traceback.format_exc()}') + + count = len(polardb_dict) + off, on = sync_polardb.w2consul('alicloud', account, region, polardb_dict) + data = {'count': count, 'update': now, 'status': 20000, 'on': on, 'off': off, + 'msg': f'polardb同步成功!总数:{count},开机:{on},关机:{off}'} + consul_kv.put_kv(f'ConsulManager/record/jobs/alicloud/{account}/polardb/{region}', data) + logger.info(f'【JOB】===>alicloud_polardb {account} {region} {data}') + except TeaException as e: + emsg = e.message.split('. ', 1)[0] + logger.error(f"【code:】{e.code}\n【message:】{e.message}\n{traceback.format_exc()}") + data = consul_kv.get_value(f'ConsulManager/record/jobs/alicloud/{account}/polardb/{region}') + if data == {}: + data = {'count': '无', 'update': f'失败{e.code}', 'status': 50000, 'msg': emsg} + else: + data['update'] = f'失败{e.code}' + data['msg'] = emsg + consul_kv.put_kv(f'ConsulManager/record/jobs/alicloud/{account}/polardb/{region}', data) + except Exception as e: + logger.error(f'{e}\n{traceback.format_exc()}') + data = {'count': '无', 'update': f'失败', 'status': 50000, 'msg': str(e)} + consul_kv.put_kv(f'ConsulManager/record/jobs/alicloud/{account}/polardb/{region}', data) + + +def mongodb(account, region): + ak, sk = consul_kv.get_aksk('alicloud', account) + now = datetime.datetime.now().strftime('%m.%d/%H:%M') + group_dict = consul_kv.get_value(f'ConsulManager/assets/alicloud/group/{account}') + + config = open_api_models.Config(access_key_id=ak, access_key_secret=sk) + config.endpoint = 'mongodb.aliyuncs.com' + if region == "ap-southeast-1": + config.endpoint = f'mongodb.{region}.aliyuncs.com' + elif region == "eu-central-1": + config.endpoint = f'mongodb.{region}.aliyuncs.com' + elif region == "us-west-1": + config.endpoint = f'mongodb.{region}.aliyuncs.com' + client = Dds20151201Client(config) # 使用MongoDB客户端 + + page_number = 1 + mongodb_dict = {} + runtime = util_models.RuntimeOptions() + try: + while True: + describe_dbinstances_request = dds_20151201_models.DescribeDBInstancesRequest( + page_size=100, + page_number=page_number, + region_id=region + ) + mongodb_info = client.describe_dbinstances_with_options(describe_dbinstances_request, runtime) + mongodb_list = mongodb_info.body.to_map()['DBInstances']["DBInstance"] + + mongodb_dict_temp = {i['DBInstanceId']: { + 'name': i.get('DBInstanceDescription', f"未命名{i['DBInstanceId']}"), + 'domain': '无', # todo + 'ip': '无', # todo + 'port': '无', # todo + 'region': region, + 'group': group_dict.get(i['ResourceGroupId'], '无'), + 'status': i['DBInstanceStatus'], # 假设MongoDB的实例有类似的status字段 + 'itype': i['DBInstanceType'], # 替换为MongoDB的实例类型 + 'ver': i['EngineVersion'], + 'exp': '-' if i.get('ExpireTime', None) is None else i.get('ExpireTime', '-T').split('T')[0], + 'cpu': '无', 'mem': '无', 'disk': '无' + } for i in mongodb_list} + mongodb_dict.update(mongodb_dict_temp) + if len(mongodb_list) < 100: + break + else: + page_number += 1 + try: + for iid in mongodb_dict.keys(): + logger.info(f'【ali_MongoDB】===> {iid}') + describe_dbinstance_attribute_request = dds_20151201_models.DescribeDBInstanceAttributeRequest(dbinstance_id=iid) + mongodb_plus_info = client.describe_dbinstance_attribute_with_options(describe_dbinstance_attribute_request, runtime) + mongodb_plus_list = mongodb_plus_info.body.to_map()['DBInstances']["DBInstance"] + mongodb_plus= {} + for i in mongodb_plus_list: + for replica_set in i['ReplicaSets']['ReplicaSet']: + if replica_set['ReplicaSetRole'] == 'Primary': + iid = i['DBInstanceId'] + mongodb_plus[iid] = { + 'port': int(replica_set['ConnectionPort']), + 'domain': replica_set['ConnectionDomain'], + 'ip': replica_set['ConnectionDomain'], + 'itype': i['DBInstanceType'], + } + break + for k, v in mongodb_plus.items(): + if k in mongodb_dict: + mongodb_dict[k].update(v) + except Exception as e: + logger.error('DescribeDBInstancesAsCsvRequest ERROR' + f'{e}\n{traceback.format_exc()}') + count = len(mongodb_dict) + off, on = sync_mongodb.w2consul('alicloud', account, region, mongodb_dict) + data = {'count': count, 'update': now, 'status': 20000, 'on': on, 'off': off, + 'msg': f'mongodb同步成功!总数:{count},开机:{on},关机:{off}'} + consul_kv.put_kv(f'ConsulManager/record/jobs/alicloud/{account}/mongodb/{region}', data) + logger.info(f'【JOB】===>alicloud_mongodb {account} {region} {data}') + except TeaException as e: + emsg = e.message.split('. ', 1)[0] + logger.error(f"【code:】{e.code}\n【message:】{e.message}\n{traceback.format_exc()}") + data = consul_kv.get_value(f'ConsulManager/record/jobs/alicloud/{account}/mongodb/{region}') + if data == {}: + data = {'count': '无', 'update': f'失败{e.code}', 'status': 50000, 'msg': emsg} + else: + data['update'] = f'失败{e.code}' + data['msg'] = emsg + consul_kv.put_kv(f'ConsulManager/record/jobs/alicloud/{account}/mongodb/{region}', data) + except Exception as e: + logger.error(f'{e}\n{traceback.format_exc()}') + data = {'count': '无', 'update': '失败', 'status': 50000, 'msg': str(e)} + consul_kv.put_kv(f'ConsulManager/record/jobs/alicloud/{account}/mongodb/{region}', data) + + +def clickhouse(account, region): + ak, sk = consul_kv.get_aksk('alicloud', account) + now = datetime.datetime.now().strftime('%m.%d/%H:%M') + group_dict = consul_kv.get_value(f'ConsulManager/assets/alicloud/group/{account}') + + config = open_api_models.Config(access_key_id=ak, access_key_secret=sk) + config.endpoint = 'clickhouse.aliyuncs.com' + if region == "ap-southeast-1": + config.endpoint = f'clickhouse.{region}.aliyuncs.com' + elif region == "eu-central-1": + config.endpoint = f'clickhouse.{region}.aliyuncs.com' + elif region == "us-west-1": + config.endpoint = f'clickhouse.{region}.aliyuncs.com' + client = clickhouse20191111Client(config) + page_number = 1 + clickhouse_dict = {} + runtime = util_models.RuntimeOptions() + try: + while True: + describe_dbinstances_request = clickhouse_20191111_models.DescribeDBClustersRequest( + page_size=100, + page_number=page_number, + region_id=region + ) + clickhouse_info = client.describe_dbclusters_with_options(describe_dbinstances_request, runtime) + clickhouse_list = clickhouse_info.body.to_map()['DBClusters']["DBCluster"] + + clickhouse_dict_temp = {i['DBClusterId']: { + 'name': i.get('DBClusterDescription', f"未命名{i['DBClusterId']}"), + 'domain': i['ConnectionString'], + 'ip': i['ConnectionString'], + 'port': 9000, + 'region': region, + 'group': group_dict.get(i.get('ResourceGroupId', '无'), '无'), + 'status': i['DBClusterStatus'], + 'itype': '社区版', + 'ver': i['DbVersion'], + 'exp': '-' if i.get('ExpireTime', None) is None else i.get('ExpireTime', '-T').split('T')[0], + 'cpu': '无', 'mem': '无', 'disk': '无' + } for i in clickhouse_list} + clickhouse_dict.update(clickhouse_dict_temp) + if len(clickhouse_list) < 100: + break + else: + page_number += 1 + + client = clickhouse20230522Client(config) + page_number = 1 + while True: + describe_dbinstances_request = clickhouse_20230522_models.DescribeDBInstancesRequest( + page_size=100, + page_number=page_number, + region_id=region + ) + clickhouse_info = client.describe_dbinstances_with_options(describe_dbinstances_request, runtime) + clickhouse_list = clickhouse_info.body.to_map()['Data']["DBInstances"] + + clickhouse_dict_temp = {i['DBInstanceId']: { + 'name': i.get('Description', f"未命名{i['DBInstanceId']}"), + 'domain': '无', + 'ip': '无', + 'port': 9000, + 'region': region, + 'group': group_dict.get(i.get('ResourceGroupId', '无'), '无'), + 'status': i['Status'], + 'itype': '企业版', + 'ver': i['EngineVersion'], + 'exp': '-' if i.get('ExpireTime', None) is None else i.get('ExpireTime', '-T').split('T')[0], + 'cpu': '无', 'mem': '无', 'disk': '无' + } for i in clickhouse_list} + clickhouse_dict.update(clickhouse_dict_temp) + if len(clickhouse_list) < 100: + break + else: + page_number += 1 + try: + for iid in clickhouse_dict.keys(): + logger.info(f'【ali_Clickhouse】===> {iid}') + if clickhouse_dict[iid].get('domain') == '无': + describe_dbinstance_attribute_request = clickhouse_20230522_models.DescribeEndpointsRequest(region_id=region, dbinstance_id=iid) + clickhouse_plus_info = client.describe_endpoints_with_options(describe_dbinstance_attribute_request, runtime) + clickhouse_plus_list = clickhouse_plus_info.body.to_map()['Data']["Endpoints"] + clickhouse_plus= {} + for i in clickhouse_plus_list: + if i['NetType'] == 'VPC': + clickhouse_plus[iid] = { + 'domain': i['VpcInstanceId'], + 'ip': i['IPAddress'], + } + break + for k, v in clickhouse_plus.items(): + if k in clickhouse_dict: + clickhouse_dict[k].update(v) + except Exception as e: + logger.error('DescribeDBInstancesAsCsvRequest ERROR' + f'{e}\n{traceback.format_exc()}') + count = len(clickhouse_dict) + off, on = sync_clickhouse.w2consul('alicloud', account, region, clickhouse_dict) + data = {'count': count, 'update': now, 'status': 20000, 'on': on, 'off': off, + 'msg': f'clickhouse同步成功!总数:{count},开机:{on},关机:{off}'} + consul_kv.put_kv(f'ConsulManager/record/jobs/alicloud/{account}/clickhouse/{region}', data) + logger.info(f'【JOB】===>alicloud_clickhouse {account} {region} {data}') + except TeaException as e: + emsg = e.message.split('. ', 1)[0] + logger.error(f"【code:】{e.code}\n【message:】{e.message}\n{traceback.format_exc()}") + data = consul_kv.get_value(f'ConsulManager/record/jobs/alicloud/{account}/clickhouse/{region}') + if data == {}: + data = {'count': '无', 'update': f'失败{e.code}', 'status': 50000, 'msg': emsg} + else: + data['update'] = f'失败{e.code}' + data['msg'] = emsg + consul_kv.put_kv(f'ConsulManager/record/jobs/alicloud/{account}/clickhouse/{region}', data) + except Exception as e: + logger.error(f'{e}\n{traceback.format_exc()}') + data = {'count': '无', 'update': '失败', 'status': 50000, 'msg': str(e)} + consul_kv.put_kv(f'ConsulManager/record/jobs/alicloud/{account}/clickhouse/{region}', data) \ No newline at end of file diff --git a/flask-consul/units/cloud/sync_clickhouse.py b/flask-consul/units/cloud/sync_clickhouse.py new file mode 100644 index 0000000..7b438f3 --- /dev/null +++ b/flask-consul/units/cloud/sync_clickhouse.py @@ -0,0 +1,77 @@ +#!/usr/bin/python3 +import requests,json +from units import consul_kv +from config import consul_token,consul_url,vendors,regions +from units.config_log import * +headers = {'X-Consul-Token': consul_token} +geturl = f'{consul_url}/agent/services' +delurl = f'{consul_url}/agent/service/deregister' +puturl = f'{consul_url}/agent/service/register' +def w2consul(vendor,account,region,clickhouse_dict): + service_name = f'{vendor}_{account}_clickhouse' + params = {'filter': f'Service == "{service_name}" and "{region}" in Tags and Meta.account == "{account}"'} + try: + consul_clickhouse_iid_list = requests.get(geturl, headers=headers, params=params).json().keys() + except: + consul_clickhouse_iid_list = [] + + #在consul中删除云厂商不存在的clickhouse + for del_clickhouse in [x for x in consul_clickhouse_iid_list if x not in clickhouse_dict.keys()]: + dereg = requests.put(f'{delurl}/{del_clickhouse}', headers=headers) + if dereg.status_code == 200: + logger.info(f"code: 20000, data: {account}-删除成功!") + else: + logger.info(f"code: 50000, data: {dereg.status_code}:{dereg.text}") + off,on = 0,0 + for k,v in clickhouse_dict.items(): + iid = k + #对consul中关机的clickhouse做标记。 + if v['status'] in ['SHUTDOWN','非运行中']: + off = off + 1 + tags = ['OFF',v['itype'],v['ver'], region] + stat = 'off' + else: + on = on + 1 + tags = ['ON',v['itype'],v['ver'],region] + stat = 'on' + custom_clickhouse = consul_kv.get_value(f'ConsulManager/assets/sync_clickhouse_custom/{iid}') + port = custom_clickhouse.get('port') + ip = custom_clickhouse.get('ip') + if port == None: + port = v['port'] + if ip == None: + ip = v['ip'] + instance = f'{ip}:{port}' + data = { + 'id': iid, + 'name': service_name, + 'Address': ip, + 'port': port, + 'tags': tags, + 'Meta': { + 'iid': iid, + 'name': v['name'], + 'region': regions[vendor].get(region,'未找到'), + 'group': v['group'], + 'instance': instance, + 'account': account, + 'itype': v['itype'], + 'vendor': vendors.get(vendor,'未找到'), + 'os': "clickhouse", + 'ver': v['ver'], + 'domain':v['domain'], + 'exp': v['exp'], + 'stat': stat, + 'team': v.get('team','无') + }, + "check": { + "tcp": f"{ip}:{port}", + "interval": "60s" + } + } + reg = requests.put(puturl, headers=headers, data=json.dumps(data)) + if reg.status_code == 200: + pass + else: + logger.info(f"{account}:code: 5000, data: {reg.status_code}:{reg.text}") + return off,on diff --git a/flask-consul/units/cloud/sync_mongodb.py b/flask-consul/units/cloud/sync_mongodb.py new file mode 100644 index 0000000..0291d65 --- /dev/null +++ b/flask-consul/units/cloud/sync_mongodb.py @@ -0,0 +1,77 @@ +#!/usr/bin/python3 +import requests,json +from units import consul_kv +from config import consul_token,consul_url,vendors,regions +from units.config_log import * +headers = {'X-Consul-Token': consul_token} +geturl = f'{consul_url}/agent/services' +delurl = f'{consul_url}/agent/service/deregister' +puturl = f'{consul_url}/agent/service/register' +def w2consul(vendor,account,region,mongodb_dict): + service_name = f'{vendor}_{account}_mongodb' + params = {'filter': f'Service == "{service_name}" and "{region}" in Tags and Meta.account == "{account}"'} + try: + consul_mongodb_iid_list = requests.get(geturl, headers=headers, params=params).json().keys() + except: + consul_mongodb_iid_list = [] + + #在consul中删除云厂商不存在的mongodb + for del_mongodb in [x for x in consul_mongodb_iid_list if x not in mongodb_dict.keys()]: + dereg = requests.put(f'{delurl}/{del_mongodb}', headers=headers) + if dereg.status_code == 200: + logger.info(f"code: 20000, data: {account}-删除成功!") + else: + logger.info(f"code: 50000, data: {dereg.status_code}:{dereg.text}") + off,on = 0,0 + for k,v in mongodb_dict.items(): + iid = k + #对consul中关机的mongodb做标记。 + if v['status'] in ['SHUTDOWN','非运行中']: + off = off + 1 + tags = ['OFF',v['itype'],v['ver'], region] + stat = 'off' + else: + on = on + 1 + tags = ['ON',v['itype'],v['ver'],region] + stat = 'on' + custom_mongodb = consul_kv.get_value(f'ConsulManager/assets/sync_mongodb_custom/{iid}') + port = custom_mongodb.get('port') + ip = custom_mongodb.get('ip') + if port == None: + port = v['port'] + if ip == None: + ip = v['ip'] + instance = f'{ip}:{port}' + data = { + 'id': iid, + 'name': service_name, + 'Address': ip, + 'port': port, + 'tags': tags, + 'Meta': { + 'iid': iid, + 'name': v['name'], + 'region': regions[vendor].get(region,'未找到'), + 'group': v['group'], + 'instance': instance, + 'account': account, + 'itype': v['itype'], + 'vendor': vendors.get(vendor,'未找到'), + 'os': "mongodb", + 'ver': v['ver'], + 'domain':v['domain'], + 'exp': v['exp'], + 'stat': stat, + 'team': v.get('team','无') + }, + "check": { + "tcp": f"{ip}:{port}", + "interval": "60s" + } + } + reg = requests.put(puturl, headers=headers, data=json.dumps(data)) + if reg.status_code == 200: + pass + else: + logger.info(f"{account}:code: 5000, data: {reg.status_code}:{reg.text}") + return off,on diff --git a/flask-consul/units/cloud/sync_polardb.py b/flask-consul/units/cloud/sync_polardb.py new file mode 100644 index 0000000..de0b116 --- /dev/null +++ b/flask-consul/units/cloud/sync_polardb.py @@ -0,0 +1,77 @@ +#!/usr/bin/python3 +import requests,json +from units import consul_kv +from config import consul_token,consul_url,vendors,regions +from units.config_log import * +headers = {'X-Consul-Token': consul_token} +geturl = f'{consul_url}/agent/services' +delurl = f'{consul_url}/agent/service/deregister' +puturl = f'{consul_url}/agent/service/register' +def w2consul(vendor,account,region,rds_dict): + service_name = f'{vendor}_{account}_polardb' + params = {'filter': f'Service == "{service_name}" and "{region}" in Tags and Meta.account == "{account}"'} + try: + consul_rds_iid_list = requests.get(geturl, headers=headers, params=params).json().keys() + except: + consul_rds_iid_list = [] + + #在consul中删除云厂商不存在的rds + for del_rds in [x for x in consul_rds_iid_list if x not in rds_dict.keys()]: + dereg = requests.put(f'{delurl}/{del_rds}', headers=headers) + if dereg.status_code == 200: + logger.info(f"code: 20000, data: {account}-删除成功!") + else: + logger.info(f"code: 50000, data: {dereg.status_code}:{dereg.text}") + off,on = 0,0 + for k,v in rds_dict.items(): + iid = k + #对consul中关机的rds做标记。 + if v['status'] in ['SHUTDOWN','非运行中']: + off = off + 1 + tags = ['OFF',v['itype'],v['ver'], region] + stat = 'off' + else: + on = on + 1 + tags = ['ON',v['itype'],v['ver'],region] + stat = 'on' + custom_rds = consul_kv.get_value(f'ConsulManager/assets/sync_rds_custom/{iid}') + port = custom_rds.get('port') + ip = custom_rds.get('ip') + if port == None: + port = v['port'] + if ip == None: + ip = v['ip'] + instance = f'{ip}:{port}' + data = { + 'id': iid, + 'name': service_name, + 'Address': ip, + 'port': port, + 'tags': tags, + 'Meta': { + 'iid': iid, + 'name': v['name'], + 'region': regions[vendor].get(region,'未找到'), + 'group': v['group'], + 'instance': instance, + 'account': account, + 'itype': v['itype'], + 'vendor': vendors.get(vendor,'未找到'), + 'os': "mysql", + 'ver': v['ver'], + 'domain':v['domain'], + 'exp': v['exp'], + 'stat': stat, + 'team': v.get('team','无') + }, + "check": { + "tcp": f"{ip}:{port}", + "interval": "60s" + } + } + reg = requests.put(puturl, headers=headers, data=json.dumps(data)) + if reg.status_code == 200: + pass + else: + logger.info(f"{account}:code: 5000, data: {reg.status_code}:{reg.text}") + return off,on diff --git a/flask-consul/units/cloud/sync_rds.py b/flask-consul/units/cloud/sync_rds.py index 2af7d87..373f040 100644 --- a/flask-consul/units/cloud/sync_rds.py +++ b/flask-consul/units/cloud/sync_rds.py @@ -42,9 +42,6 @@ def w2consul(vendor,account,region,rds_dict): if ip == None: ip = v['ip'] instance = f'{ip}:{port}' - if vendor == 'alicloud' and iid in consul_rds_iid_list and v['cpu'] == '无': - continue - data = { 'id': iid, 'name': service_name, @@ -60,6 +57,7 @@ def w2consul(vendor,account,region,rds_dict): 'account': account, 'itype': v['itype'], 'vendor': vendors.get(vendor,'未找到'), + 'os': "mysql", 'disk': v['disk'], 'cpu': v['cpu'], 'mem': v['mem'], diff --git a/flask-consul/units/cloud/sync_redis.py b/flask-consul/units/cloud/sync_redis.py index 6ea15f9..a767250 100644 --- a/flask-consul/units/cloud/sync_redis.py +++ b/flask-consul/units/cloud/sync_redis.py @@ -57,6 +57,7 @@ def w2consul(vendor,account,region,redis_dict): 'account': account, 'itype': v['itype'], 'vendor': vendors.get(vendor,'未找到'), + 'os': "redis", 'mem': v['mem'], 'ver': v['ver'], 'ip':v['ip'], diff --git a/flask-consul/units/config_log.py b/flask-consul/units/config_log.py index 45b0189..60779d9 100644 --- a/flask-consul/units/config_log.py +++ b/flask-consul/units/config_log.py @@ -1,5 +1,3 @@ from config import log_level import sys from loguru import logger -logger.remove() -logger.add(sys.stdout,format='{time:HH:mm:ss} | {level} | {message}',level=log_level) diff --git a/flask-consul/units/jms/sync_jms.py b/flask-consul/units/jms/sync_jms.py index ea0c066..1932db2 100755 --- a/flask-consul/units/jms/sync_jms.py +++ b/flask-consul/units/jms/sync_jms.py @@ -2,14 +2,26 @@ import datetime,requests,json,traceback from units import consul_kv,consul_manager,myaes from units.config_log import * -def exist_ssh_port(port,protocols): - for protocol in protocols: - if protocol.get('name') == 'ssh': - return protocol.get('port', port) if protocol.get('port') != port else port - return port +resource_type = ["ecs", "redis", "rds", "mongodb", "polardb", "clickhouse"] -#创建node -def create_node(jms_url,headers,now,node_id,cloud,account): + +# 创建节点 +def create_node(jms_url,headers, node_id, account): + node_url = f"{jms_url}/api/v1/assets/nodes/{node_id}/children/" + jms_node_list = requests.request("GET", node_url, headers=headers).json() + for resource in resource_type: + if resource not in [i['value'] for i in jms_node_list]: + response = requests.request("POST", node_url, headers=headers, data=json.dumps({'value': resource})) + if response.status_code != 201: + logger.error(f' 【JMS】创建{account}/{resource}分组失败,可能的原因:JumpServer URL 有重定向,请使用直连地址。') + logger.debug(f' 【JMS】{account}新增组===>{resource},{response.status_code}') + reget_node_list = requests.request("GET", node_url, headers=headers).json() + new_node_dict = {i['value']: i['id'] for i in reget_node_list} + return new_node_dict + + +# 创建资源组node +def create_resource_node(jms_url,headers,now,node_id,cloud,account): node_url = f"{jms_url}/api/v1/assets/nodes/{node_id}/children/" logger.debug(f'{node_url}==>{headers}') jms_node_list = requests.request("GET", node_url, headers=headers).json() @@ -30,7 +42,7 @@ def create_node(jms_url,headers,now,node_id,cloud,account): new_node_dict = {i['value']:i['id'] for i in reget_node_list} return new_node_dict -def update_jms_ecs(jms_ver,jms_url,headers,new_node_dict,node_id,cloud,account,ecs_info,custom_ecs_info): +def update_jms(jms_ver,jms_url,headers,new_node_dict,node_id,cloud,account,ecs_info,custom_ecs_info,resource_type): #比较云主机与JMS中对应node的主机列表,删除jms中多余的主机 ecs_url = f"{jms_url}/api/v1/assets/assets/" reget_ecs_list = requests.request("GET", f'{ecs_url}?node={node_id}', headers=headers).json() @@ -39,9 +51,9 @@ def update_jms_ecs(jms_ver,jms_url,headers,new_node_dict,node_id,cloud,account,e except: jms_ecs_dict = {i.get('ip',i.get('address','IPNOTFOUND')):{'name':i.get('hostname',i.get('name','NAMENOTFOUND')),'id':i['id'],'protocols': i['protocols'],'comment':i['comment'],'node':i['nodes'][0]} for i in reget_ecs_list} - ecs_list = consul_manager.get_instances(f'{cloud}_{account}_ecs')['instances'] + ecs_list = consul_manager.get_instances(f'{cloud}_{account}_{resource_type}')['instances'] ecs_ip_dict = {i['address']:i['meta'][0]['name'] for i in ecs_list} - ecs_dict = {i['ID']:{'name':i['meta'][0]['name'],'ip':i['address'],'ent':i['meta'][0]['group'],'ostype':i['meta'][0]['os'],'region':i['meta'][0]['region'],'vendor':i['meta'][0]['vendor']} for i in ecs_list} + ecs_dict = {i['ID']:{'name':i['meta'][0]['name'],'ip':i['address'],'ent':i['meta'][0]['group'],'ostype':i['meta'][0]['os'],'region':i['meta'][0]['region'],'vendor':i['meta'][0]['vendor'],'ver':i['meta'][0].get('ver')} for i in ecs_list} del_ecs_list = [v['id'] for k,v in jms_ecs_dict.items() if k not in [i['ip'] for i in ecs_dict.values()]] for del_ecs in del_ecs_list: response = requests.request("DELETE", f'{ecs_url}{del_ecs}/', headers=headers) @@ -67,12 +79,10 @@ def update_jms_ecs(jms_ver,jms_url,headers,new_node_dict,node_id,cloud,account,e if jms_ver == 'V3': ecs_url = f"{jms_url}/api/v1/assets/hosts/" proto,proto_port = protocols[0].split('/') - port = exist_ssh_port(proto_port, jms_ecs_dict.get(ip, {}).get("protocols", [])) + port = proto_port payload = { "address": ip, "name": iname, - "protocols": [{"name": proto,"port": port}, {"name": "sftp", "port": port}], - "platform": '5' if platform == 'Windows' else '1', "is_active": True, "domain": "", "accounts":[{"template": admin_user.strip()}], @@ -84,17 +94,52 @@ def update_jms_ecs(jms_ver,jms_url,headers,new_node_dict,node_id,cloud,account,e "ip": ip, "hostname": iname, "protocols": protocols, - "platform": platform, "is_active": True, "domain": "", "admin_user": admin_user.strip(), "nodes": [nodes], "comment": comment } + if platform == 'Linux': + payload["platform"] = '1' + payload["protocols"] = [{"name": proto, "port": port}, {"name": "sftp", "port": port}] + elif platform == 'Windows': + payload["platform"] = "5" + payload["protocols"] = [{"name": proto, "port": port}] + elif platform == 'Redis': + ecs_url = f"{jms_url}/api/v1/assets/databases/" + payload["protocols"] = [{"name": proto, "port": port}] + payload.update({"db_name": "0", "use_ssl": False, "allow_invalid_cert": False}) + # if float(v['ver']) < 6: + # payload["platform"] = '24' + # else: + payload["platform"] = "25" + elif platform == 'Mysql': + ecs_url = f"{jms_url}/api/v1/assets/databases/" + payload["platform"] = '17' + payload["protocols"] = [{"name": proto, "port": port}] + elif platform == 'Mongodb': + ecs_url = f"{jms_url}/api/v1/assets/databases/" + payload["platform"] = '33' + payload["protocols"] = [{"name": proto, "port": port}] + payload.update({"db_name": "admin"}) + elif platform == 'Clickhouse': + ecs_url = f"{jms_url}/api/v1/assets/databases/" + payload["platform"] = '22' + payload["protocols"] = [{"name": proto, "port": port}] + payload.update({"db_name": "default"}) + else: + logger.error(f"未匹配到{platform}!") + continue try: if ip in jms_ecs_dict.keys(): jms_group = '无' if jms_ecs_dict[ip]['node'].split('/')[-1] == '未分组' else jms_ecs_dict[ip]['node'].split('/')[-1] - if jms_ecs_dict[ip]['name'] != iname or jms_group != v['ent']: + is_update = False + for p in jms_ecs_dict.get(ip, {}).get("protocols", []): + if p['name'] == proto and int(p['port']) != int(port): + is_update = True + break + if jms_ecs_dict[ip]['name'] != iname or jms_group != v['ent'] or is_update: response = requests.request("PUT", f"{ecs_url}{jms_ecs_dict[ip]['id']}/", headers=headers, data = json.dumps(payload)) logger.info(f" 【JMS】update:主机名:{response.json().get('hostname',response.json())},{response.status_code}") else: @@ -105,6 +150,7 @@ def update_jms_ecs(jms_ver,jms_url,headers,new_node_dict,node_id,cloud,account,e logger.error(f'{response.json()}') return ecs_ip_dict + #从JMS中删除IP重复的主机 def del_jms_repip(jms_url,headers,node_id,ecs_ip_dict): ecs_url = f"{jms_url}/api/v1/assets/assets/" @@ -125,7 +171,7 @@ def del_jms_repip(jms_url,headers,node_id,ecs_ip_dict): logger.info(f" 【JMS】删除IP重复且名称不在ECS列表的主机:{j['name']},{j['ip']},{response.status_code}") #从JMS中删除没有主机的组 -def del_node(jms_url,headers,now,node_id,cloud,account): +def del_node(jms_url,headers,now,node_id,cloud,account, main_node_id): node_tree_url = f"{jms_url}/api/v1/assets/nodes/children/tree/?id={node_id}" jms_node_list = requests.request("GET", node_tree_url, headers=headers).json() for i in jms_node_list: @@ -136,16 +182,17 @@ def del_node(jms_url,headers,now,node_id,cloud,account): del_node_url = f"{jms_url}/api/v1/assets/nodes/{i['meta']['data']['id']}/" response = requests.request("DELETE", del_node_url, headers=headers) logger.debug(f" 【JMS】删除空组===>{i['name']},{response.status_code}") - ecs_count_url = f"{jms_url}/api/v1/assets/assets/?node={node_id}&limit=1&offset=1" + ecs_count_url = f"{jms_url}/api/v1/assets/assets/?node={main_node_id}&limit=1&offset=1" ecs_count = requests.request("GET", ecs_count_url, headers=headers).json()['count'] data = {'count':ecs_count,'update':now,'status':20000,'msg':f'同步资源成功!总数:{ecs_count}'} consul_kv.put_kv(f'ConsulManager/record/jms/{cloud}/{account}', data) return 'ok' + def run(cloud,account): now = datetime.datetime.now().strftime('%m%d/%H:%M') logger.info(f'【JOB】===>{cloud},{account},JMS同步开始') - node_id = consul_kv.get_value(f'ConsulManager/jms/{cloud}/{account}/node_id')['node_id'] + main_node_id = consul_kv.get_value(f'ConsulManager/jms/{cloud}/{account}/node_id')['node_id'] temp_ecs_info = consul_kv.get_value(f'ConsulManager/jms/{cloud}/{account}/ecs_info') ecs_info = consul_kv.get_value(f'ConsulManager/jms/ecs_info') if temp_ecs_info == {} else temp_ecs_info temp_custom_ecs_info = consul_kv.get_value(f'ConsulManager/jms/{cloud}/{account}/custom_ecs_info') @@ -156,9 +203,17 @@ def run(cloud,account): jms_ver = jms.get('ver','V2') token = myaes.decrypt(jms.get('token')) headers = {'Content-Type': 'application/json','Authorization': f"Token {token}"} + type_nodes = create_node(jms_url, headers, main_node_id, account) + + for t, node_id in type_nodes.items(): + new_node_dict = create_resource_node(jms_url,headers,now,node_id,cloud,account) + if t in resource_type: + ecs_ip_dict = update_jms(jms_ver,jms_url,headers,new_node_dict,node_id,cloud,account,ecs_info,custom_ecs_info, t) + del_jms_repip(jms_url,headers,node_id,ecs_ip_dict) + del_node(jms_url,headers,now,node_id,cloud,account,main_node_id) + else: + logger.error(f"{t} not exist!") - new_node_dict = create_node(jms_url,headers,now,node_id,cloud,account) - ecs_ip_dict = update_jms_ecs(jms_ver,jms_url,headers,new_node_dict,node_id,cloud,account,ecs_info,custom_ecs_info) - del_jms_repip(jms_url,headers,node_id,ecs_ip_dict) - del_node(jms_url,headers,now,node_id,cloud,account) logger.info(f'【JOB】===>{cloud},{account},JMS同步完成') + + diff --git a/flask-consul/units/selfclickhouse_manager.py b/flask-consul/units/selfclickhouse_manager.py new file mode 100644 index 0000000..026e43a --- /dev/null +++ b/flask-consul/units/selfclickhouse_manager.py @@ -0,0 +1,77 @@ +import requests,json,re +from config import consul_token,consul_url +from units.config_log import * +headers = {'X-Consul-Token': consul_token} + +def get_all_list(vendor,account,region,group): + vendor = f'and Meta.vendor=="{vendor}"' if vendor != '' else f'and Meta.vendor != ""' + account = f'and Meta.account=="{account}"' if account != '' else f'and Meta.account != ""' + region = f'and Meta.region=="{region}"' if region != '' else f'and Meta.region != ""' + group = f'and Meta.group=="{group}"' if group != '' else f'and Meta.group != ""' + url = f'{consul_url}/agent/services?filter=Service == selfrds_exporter {vendor} {account} {region} {group}' + response = requests.get(url, headers=headers) + if response.status_code == 200: + info = response.json() + all_list = [i['Meta'] for i in info.values()] + vendor_list = sorted(list(set([i['vendor'] for i in all_list]))) + account_list = sorted(list(set([i['account'] for i in all_list]))) + region_list = sorted(list(set([i['region'] for i in all_list]))) + group_list = sorted(list(set([i['group'] for i in all_list]))) + return {'code': 20000,'all_list':all_list,'vendor_list':vendor_list, + 'account_list':account_list,'region_list':region_list,'group_list':group_list} + else: + logger.error(f"{response.status_code}:{response.text}") + return {'code': 50000, 'data': f'{response.status_code}:{response.text}'} + +def get_service(): + response = requests.get(f'{consul_url}/agent/services?filter=Service == selfrds_exporter', headers=headers) + if response.status_code == 200: + info = response.json() + all_list = [i['Meta'] for i in info.values()] + vendor_list = sorted(list(set([i['vendor'] for i in all_list]))) + account_list = sorted(list(set([i['account'] for i in all_list]))) + region_list = sorted(list(set([i['region'] for i in all_list]))) + group_list = sorted(list(set([i['group'] for i in all_list]))) + return {'code': 20000,'all_list':all_list,'vendor_list':vendor_list, + 'account_list':account_list,'region_list':region_list,'group_list':group_list} + else: + logger.error(f"{response.status_code}:{response.text}") + return {'code': 50000, 'data': f'{response.status_code}:{response.text}'} + +def add_service(vendor,account,region,group,name,ip,port,os): + if port is None or name is None: + return {"code": 50000, "data": f"名称或IP不能为空!"} + sid = f"{vendor}/{account}/{region}/{group}@{name}".strip() + #sid = eval(repr(sid).replace('\\t','').replace('\\n','')) + sid = re.sub('[[ \]`~!\\\#$^&*=|"{}\':;?\t\n]','_',sid)+'@rds' + instance = f'{ip}:{port}' + if '//' in sid or sid.startswith('/'): + return {"code": 50000, "data": f"服务ID【{sid}】首尾不能包含'/',并且不能包含两个连续的'/'"} + data = { + "id": sid, + "name": 'selfrds_exporter', + 'Address': ip, + 'port': int(port), + "tags": [vendor,os], + "Meta": {'vendor':vendor,'account':account,'region':region,'group':group, + 'name':name,'instance':instance,'os':os}, + "check": {"tcp": instance,"interval": "60s"} + } + reg = requests.put(f'{consul_url}/agent/service/register', headers=headers, data=json.dumps(data)) + if reg.status_code == 200: + return {"code": 20000, "data": f"【{sid}】增加成功!"} + else: + logger.error(f"{reg.status_code}【{sid}】{reg.text}") + return {"code": 50000, "data": f"{reg.status_code}【{sid}】{reg.text}"} + +def del_service(vendor,account,region,group,name): + sid = f"{vendor}/{account}/{region}/{group}@{name}".strip() + sid = re.sub('[[ \]`~!\\\#$^&*=|"{}\':;?\t\n]','_',sid)+'@rds' + reg = requests.put(f'{consul_url}/agent/service/deregister/{sid}', headers=headers) + if reg.status_code == 200: + logger.debug(f"【{sid}】删除成功!") + return {"code": 20000, "data": f"【{sid}】删除成功!"} + else: + logger.error(f"{reg.status_code}【{sid}】{reg.text}") + return {"code": 50000, "data": f"{reg.status_code}【{sid}】{reg.text}"} + diff --git a/flask-consul/units/selfmongodb_manager.py b/flask-consul/units/selfmongodb_manager.py new file mode 100644 index 0000000..026e43a --- /dev/null +++ b/flask-consul/units/selfmongodb_manager.py @@ -0,0 +1,77 @@ +import requests,json,re +from config import consul_token,consul_url +from units.config_log import * +headers = {'X-Consul-Token': consul_token} + +def get_all_list(vendor,account,region,group): + vendor = f'and Meta.vendor=="{vendor}"' if vendor != '' else f'and Meta.vendor != ""' + account = f'and Meta.account=="{account}"' if account != '' else f'and Meta.account != ""' + region = f'and Meta.region=="{region}"' if region != '' else f'and Meta.region != ""' + group = f'and Meta.group=="{group}"' if group != '' else f'and Meta.group != ""' + url = f'{consul_url}/agent/services?filter=Service == selfrds_exporter {vendor} {account} {region} {group}' + response = requests.get(url, headers=headers) + if response.status_code == 200: + info = response.json() + all_list = [i['Meta'] for i in info.values()] + vendor_list = sorted(list(set([i['vendor'] for i in all_list]))) + account_list = sorted(list(set([i['account'] for i in all_list]))) + region_list = sorted(list(set([i['region'] for i in all_list]))) + group_list = sorted(list(set([i['group'] for i in all_list]))) + return {'code': 20000,'all_list':all_list,'vendor_list':vendor_list, + 'account_list':account_list,'region_list':region_list,'group_list':group_list} + else: + logger.error(f"{response.status_code}:{response.text}") + return {'code': 50000, 'data': f'{response.status_code}:{response.text}'} + +def get_service(): + response = requests.get(f'{consul_url}/agent/services?filter=Service == selfrds_exporter', headers=headers) + if response.status_code == 200: + info = response.json() + all_list = [i['Meta'] for i in info.values()] + vendor_list = sorted(list(set([i['vendor'] for i in all_list]))) + account_list = sorted(list(set([i['account'] for i in all_list]))) + region_list = sorted(list(set([i['region'] for i in all_list]))) + group_list = sorted(list(set([i['group'] for i in all_list]))) + return {'code': 20000,'all_list':all_list,'vendor_list':vendor_list, + 'account_list':account_list,'region_list':region_list,'group_list':group_list} + else: + logger.error(f"{response.status_code}:{response.text}") + return {'code': 50000, 'data': f'{response.status_code}:{response.text}'} + +def add_service(vendor,account,region,group,name,ip,port,os): + if port is None or name is None: + return {"code": 50000, "data": f"名称或IP不能为空!"} + sid = f"{vendor}/{account}/{region}/{group}@{name}".strip() + #sid = eval(repr(sid).replace('\\t','').replace('\\n','')) + sid = re.sub('[[ \]`~!\\\#$^&*=|"{}\':;?\t\n]','_',sid)+'@rds' + instance = f'{ip}:{port}' + if '//' in sid or sid.startswith('/'): + return {"code": 50000, "data": f"服务ID【{sid}】首尾不能包含'/',并且不能包含两个连续的'/'"} + data = { + "id": sid, + "name": 'selfrds_exporter', + 'Address': ip, + 'port': int(port), + "tags": [vendor,os], + "Meta": {'vendor':vendor,'account':account,'region':region,'group':group, + 'name':name,'instance':instance,'os':os}, + "check": {"tcp": instance,"interval": "60s"} + } + reg = requests.put(f'{consul_url}/agent/service/register', headers=headers, data=json.dumps(data)) + if reg.status_code == 200: + return {"code": 20000, "data": f"【{sid}】增加成功!"} + else: + logger.error(f"{reg.status_code}【{sid}】{reg.text}") + return {"code": 50000, "data": f"{reg.status_code}【{sid}】{reg.text}"} + +def del_service(vendor,account,region,group,name): + sid = f"{vendor}/{account}/{region}/{group}@{name}".strip() + sid = re.sub('[[ \]`~!\\\#$^&*=|"{}\':;?\t\n]','_',sid)+'@rds' + reg = requests.put(f'{consul_url}/agent/service/deregister/{sid}', headers=headers) + if reg.status_code == 200: + logger.debug(f"【{sid}】删除成功!") + return {"code": 20000, "data": f"【{sid}】删除成功!"} + else: + logger.error(f"{reg.status_code}【{sid}】{reg.text}") + return {"code": 50000, "data": f"{reg.status_code}【{sid}】{reg.text}"} + diff --git a/flask-consul/units/selfpolardb_manager.py b/flask-consul/units/selfpolardb_manager.py new file mode 100644 index 0000000..026e43a --- /dev/null +++ b/flask-consul/units/selfpolardb_manager.py @@ -0,0 +1,77 @@ +import requests,json,re +from config import consul_token,consul_url +from units.config_log import * +headers = {'X-Consul-Token': consul_token} + +def get_all_list(vendor,account,region,group): + vendor = f'and Meta.vendor=="{vendor}"' if vendor != '' else f'and Meta.vendor != ""' + account = f'and Meta.account=="{account}"' if account != '' else f'and Meta.account != ""' + region = f'and Meta.region=="{region}"' if region != '' else f'and Meta.region != ""' + group = f'and Meta.group=="{group}"' if group != '' else f'and Meta.group != ""' + url = f'{consul_url}/agent/services?filter=Service == selfrds_exporter {vendor} {account} {region} {group}' + response = requests.get(url, headers=headers) + if response.status_code == 200: + info = response.json() + all_list = [i['Meta'] for i in info.values()] + vendor_list = sorted(list(set([i['vendor'] for i in all_list]))) + account_list = sorted(list(set([i['account'] for i in all_list]))) + region_list = sorted(list(set([i['region'] for i in all_list]))) + group_list = sorted(list(set([i['group'] for i in all_list]))) + return {'code': 20000,'all_list':all_list,'vendor_list':vendor_list, + 'account_list':account_list,'region_list':region_list,'group_list':group_list} + else: + logger.error(f"{response.status_code}:{response.text}") + return {'code': 50000, 'data': f'{response.status_code}:{response.text}'} + +def get_service(): + response = requests.get(f'{consul_url}/agent/services?filter=Service == selfrds_exporter', headers=headers) + if response.status_code == 200: + info = response.json() + all_list = [i['Meta'] for i in info.values()] + vendor_list = sorted(list(set([i['vendor'] for i in all_list]))) + account_list = sorted(list(set([i['account'] for i in all_list]))) + region_list = sorted(list(set([i['region'] for i in all_list]))) + group_list = sorted(list(set([i['group'] for i in all_list]))) + return {'code': 20000,'all_list':all_list,'vendor_list':vendor_list, + 'account_list':account_list,'region_list':region_list,'group_list':group_list} + else: + logger.error(f"{response.status_code}:{response.text}") + return {'code': 50000, 'data': f'{response.status_code}:{response.text}'} + +def add_service(vendor,account,region,group,name,ip,port,os): + if port is None or name is None: + return {"code": 50000, "data": f"名称或IP不能为空!"} + sid = f"{vendor}/{account}/{region}/{group}@{name}".strip() + #sid = eval(repr(sid).replace('\\t','').replace('\\n','')) + sid = re.sub('[[ \]`~!\\\#$^&*=|"{}\':;?\t\n]','_',sid)+'@rds' + instance = f'{ip}:{port}' + if '//' in sid or sid.startswith('/'): + return {"code": 50000, "data": f"服务ID【{sid}】首尾不能包含'/',并且不能包含两个连续的'/'"} + data = { + "id": sid, + "name": 'selfrds_exporter', + 'Address': ip, + 'port': int(port), + "tags": [vendor,os], + "Meta": {'vendor':vendor,'account':account,'region':region,'group':group, + 'name':name,'instance':instance,'os':os}, + "check": {"tcp": instance,"interval": "60s"} + } + reg = requests.put(f'{consul_url}/agent/service/register', headers=headers, data=json.dumps(data)) + if reg.status_code == 200: + return {"code": 20000, "data": f"【{sid}】增加成功!"} + else: + logger.error(f"{reg.status_code}【{sid}】{reg.text}") + return {"code": 50000, "data": f"{reg.status_code}【{sid}】{reg.text}"} + +def del_service(vendor,account,region,group,name): + sid = f"{vendor}/{account}/{region}/{group}@{name}".strip() + sid = re.sub('[[ \]`~!\\\#$^&*=|"{}\':;?\t\n]','_',sid)+'@rds' + reg = requests.put(f'{consul_url}/agent/service/deregister/{sid}', headers=headers) + if reg.status_code == 200: + logger.debug(f"【{sid}】删除成功!") + return {"code": 20000, "data": f"【{sid}】删除成功!"} + else: + logger.error(f"{reg.status_code}【{sid}】{reg.text}") + return {"code": 50000, "data": f"{reg.status_code}【{sid}】{reg.text}"} + diff --git a/flask-consul/views/clickhouse.py b/flask-consul/views/clickhouse.py new file mode 100644 index 0000000..20500a0 --- /dev/null +++ b/flask-consul/views/clickhouse.py @@ -0,0 +1,93 @@ +from flask import Blueprint +from flask_restful import reqparse, Resource, Api +from flask_apscheduler import APScheduler +import traceback +#import sys +#sys.path.append("..") +from units import token_auth,consul_kv,gen_config,consul_svc +from units.config_log import * +blueprint = Blueprint('clickhouse',__name__) +api = Api(blueprint) + +parser = reqparse.RequestParser() +parser.add_argument('job_id',type=str) +parser.add_argument('services_dict',type=dict) +parser.add_argument('cst_clickhouse_dict',type=dict) +parser.add_argument('iid',type=str) +parser.add_argument('jobclickhouse_name',type=str) +parser.add_argument('checked',type=str) + +class Clickhouse(Resource): + decorators = [token_auth.auth.login_required] + def get(self, stype): + job_id = parser.parse_args()['job_id'] + if stype == 'jobclickhouse': + jobclickhouse = consul_kv.get_keys_list('ConsulManager/jobs') + jobclickhouse_list = [i.split('/jobs/')[1] for i in jobclickhouse if '/clickhouse/' in i] + return {'code': 20000,'jobclickhouse':jobclickhouse_list} + elif stype == 'clickhouse_services': + jobecs = consul_kv.get_keys_list('ConsulManager/jobs') + jobecs_list = [i.split('/jobs/')[1] for i in jobecs if '/clickhouse/' in i] + services_list = [] + for i in jobecs_list: + serivces = i.split("/") + services_list.append(f'{serivces[0]}_{serivces[1]}_{serivces[2]}') + return {'code': 20000,'services_list': sorted(set(services_list))} + elif stype == 'clickhouserules': + return gen_config.get_clickhouserules() + elif stype == 'cstclickhouseconf': + args = parser.parse_args() + iid = args['iid'] + cst_clickhouse_config = consul_kv.get_value(f'ConsulManager/assets/sync_clickhouse_custom/{iid}') + cst_clickhouse_config.update({'iid': iid,'ipswitch': False,'portswitch': False}) + if 'ip' in cst_clickhouse_config and cst_clickhouse_config['ip'] != '': + cst_clickhouse_config['ipswitch'] = True + if 'port' in cst_clickhouse_config and cst_clickhouse_config['port'] != '': + cst_clickhouse_config['portswitch'] = True + return {'code': 20000, 'cst_clickhouse': cst_clickhouse_config} + elif stype == 'cstclickhouselist': + args = parser.parse_args() + jobclickhouse_name = args['jobclickhouse_name'] + checked = args['checked'] + cst_clickhouse_dict = consul_kv.get_kv_dict('ConsulManager/assets/sync_clickhouse_custom/') + cst_clickhouse_keylist = [k.split('/')[-1] for k,v in cst_clickhouse_dict.items() if v != {}] + clickhouse_info = consul_kv.get_res_services(jobclickhouse_name) + if checked == 'false': + return clickhouse_info + else: + cst_clickhouse_list = [i for i in clickhouse_info['res_list'] if i['iid'] in cst_clickhouse_keylist] + return {'code': 20000, 'res_list': cst_clickhouse_list} + + def post(self, stype): + if stype == 'clickhousepconfig': + args = parser.parse_args() + services_dict = args['services_dict'] + return gen_config.clickhouse_config(services_dict['jobclickhouse_list'],services_dict['cm_exporter'],services_dict['services_list'],services_dict['exporter']) + elif stype == 'cstclickhouse': + args = parser.parse_args() + cst_clickhouse_dict = args['cst_clickhouse_dict'] + consul_clickhouse_cst = {} + iid = cst_clickhouse_dict['iid'] + try: + sid_dict = consul_svc.get_sid(iid)['instance'] + if cst_clickhouse_dict['portswitch'] and cst_clickhouse_dict['port'] != '': + consul_clickhouse_cst['port'] = int(cst_clickhouse_dict['port']) + sid_dict['Port'] = consul_clickhouse_cst['port'] + if cst_clickhouse_dict['ipswitch'] and cst_clickhouse_dict['ip'] != '': + consul_clickhouse_cst['ip'] = cst_clickhouse_dict['ip'] + sid_dict['Address'] = consul_clickhouse_cst['ip'] + consul_kv.put_kv(f'ConsulManager/assets/sync_clickhouse_custom/{iid}',consul_clickhouse_cst) + del sid_dict['Weights'] + del sid_dict['ContentHash'] + del sid_dict['Datacenter'] + sid_dict['name'] = sid_dict.pop('Service') + sid_dict['Meta']['instance'] = f"{sid_dict['Address']}:{sid_dict['Port']}" + sid_dict["check"] = { "tcp": sid_dict['Meta']['instance'],"interval": "60s" } + consul_svc.del_sid(iid) + consul_svc.add_sid(sid_dict) + return {'code': 20000, 'data': '自定义实例信息修改成功!'} + except Exception as e: + logger.error(f'{e}\n{traceback.format_exc()}') + return {'code': 50000, "data": '提交自定义实例信息格式错误!'} + +api.add_resource(Clickhouse, '/api/clickhouse/') diff --git a/flask-consul/views/edit_cloud.py b/flask-consul/views/edit_cloud.py index f9f794e..5f0e529 100644 --- a/flask-consul/views/edit_cloud.py +++ b/flask-consul/views/edit_cloud.py @@ -34,7 +34,8 @@ class Edit(Resource): account = args['account'] region = args['region'] restype = ['group'] - interval = {'proj_interval': 60, 'ecs_interval': 10, 'rds_interval': 20, 'redis_interval': 20} + interval = {'proj_interval': 60, 'ecs_interval': 10, 'rds_interval': 20, 'redis_interval': 20, + 'polardb_interval': 20, 'mongodb_interval': 20, 'clickhouse_interval': 20} isextip = False for i in job_list: if f'{vendor}/{account}/group' == i['id']: @@ -49,6 +50,15 @@ class Edit(Resource): elif f'{vendor}/{account}/redis/{region}' == i['id']: restype.append('redis') interval['redis_interval'] = i['minutes'] + elif f'{vendor}/{account}/polardb/{region}' == i['id']: + restype.append('polardb') + interval['polardb_interval'] = i['minutes'] + elif f'{vendor}/{account}/mongodb/{region}' == i['id']: + restype.append('mongodb') + interval['mongodb_interval'] = i['minutes'] + elif f'{vendor}/{account}/clickhouse/{region}' == i['id']: + restype.append('clickhouse') + interval['clickhouse_interval'] = i['minutes'] return {'code': 20000, 'restype': restype, 'interval': interval, 'isextip': isextip} def post(self,stype): job_list = list(consul_kv.get_kv_dict(f'ConsulManager/jobs').values()) @@ -63,7 +73,10 @@ class Edit(Resource): proj_interval = int(editjob_dict['proj_interval']) ecs_interval = int(editjob_dict['ecs_interval']) rds_interval = int(editjob_dict['rds_interval']) + polardb_interval = int(editjob_dict['polardb_interval']) + mongodb_interval = int(editjob_dict['mongodb_interval']) redis_interval = int(editjob_dict['redis_interval']) + clickhouse_interval = int(editjob_dict['clickhouse_interval']) logger.info(f'{editjob_dict}') if editjob_dict['akskswitch']: ak = editjob_dict['ak'] @@ -79,7 +92,10 @@ class Edit(Resource): ecs_jobid = f'{vendor}/{account}/ecs/{region}' rds_jobid = f'{vendor}/{account}/rds/{region}' + polardb_jobid = f'{vendor}/{account}/polardb/{region}' + mongodb_jobid = f'{vendor}/{account}/mongodb/{region}' redis_jobid = f'{vendor}/{account}/redis/{region}' + clickhouse_jobid = f'{vendor}/{account}/clickhouse/{region}' if 'ecs' in restype: isecs = [x for x in job_list if x['id'] == f'{vendor}/{account}/ecs/{region}'] if len(isecs) == 1: @@ -132,6 +148,28 @@ class Edit(Resource): except: pass + if 'polardb' in restype: + ispolardb = [x for x in job_list if x['id'] == f'{vendor}/{account}/polardb/{region}'] + if len(ispolardb) == 1: + if polardb_interval != ispolardb[0]['minutes']: + ispolardb[0]['minutes'] = polardb_interval + consul_kv.put_kv(f'ConsulManager/jobs/{polardb_jobid}',ispolardb[0]) + modjob_interval(polardb_jobid,polardb_interval) + else: + job_func = f"__main__:{vendor}.polardb" + job_args = [account,region] + job_interval = polardb_interval + addjob(polardb_jobid, job_func, job_args, job_interval) + job_dict = {'id':polardb_jobid,'func':job_func,'args':job_args,'minutes':job_interval, + "trigger": "interval","replace_existing": True} + consul_kv.put_kv(f'ConsulManager/jobs/{polardb_jobid}',job_dict) + else: + try: + consul_kv.del_key(f'ConsulManager/jobs/{polardb_jobid}') + deljob(polardb_jobid) + except: + pass + if 'redis' in restype: isredis = [x for x in job_list if x['id'] == f'{vendor}/{account}/redis/{region}'] if len(isredis) == 1: @@ -154,5 +192,49 @@ class Edit(Resource): except: pass + if 'mongodb' in restype: + ismongodb = [x for x in job_list if x['id'] == f'{vendor}/{account}/mongodb/{region}'] + if len(ismongodb) == 1: + if mongodb_interval != ismongodb[0]['minutes']: + ismongodb[0]['minutes'] = mongodb_interval + consul_kv.put_kv(f'ConsulManager/jobs/{mongodb_jobid}',ismongodb[0]) + modjob_interval(mongodb_jobid,mongodb_interval) + else: + job_func = f"__main__:{vendor}.mongodb" + job_args = [account,region] + job_interval = mongodb_interval + addjob(mongodb_jobid, job_func, job_args, job_interval) + job_dict = {'id':mongodb_jobid,'func':job_func,'args':job_args,'minutes':job_interval, + "trigger": "interval","replace_existing": True} + consul_kv.put_kv(f'ConsulManager/jobs/{mongodb_jobid}',job_dict) + else: + try: + consul_kv.del_key(f'ConsulManager/jobs/{mongodb_jobid}') + deljob(mongodb_jobid) + except: + pass + + if 'clickhouse' in restype: + isclickhouse = [x for x in job_list if x['id'] == f'{vendor}/{account}/clickhouse/{region}'] + if len(isclickhouse) == 1: + if clickhouse_interval != isclickhouse[0]['minutes']: + isclickhouse[0]['minutes'] = clickhouse_interval + consul_kv.put_kv(f'ConsulManager/jobs/{clickhouse_jobid}',isclickhouse[0]) + modjob_interval(clickhouse_jobid,clickhouse_interval) + else: + job_func = f"__main__:{vendor}.clickhouse" + job_args = [account,region] + job_interval = clickhouse_interval + addjob(clickhouse_jobid, job_func, job_args, job_interval) + job_dict = {'id':clickhouse_jobid,'func':job_func,'args':job_args,'minutes':job_interval, + "trigger": "interval","replace_existing": True} + consul_kv.put_kv(f'ConsulManager/jobs/{clickhouse_jobid}',job_dict) + else: + try: + consul_kv.del_key(f'ConsulManager/jobs/{clickhouse_jobid}') + deljob(clickhouse_jobid) + except: + pass + return {'code': 20000, 'data': f'{vendor}/{account}/{region}:编辑成功!'} api.add_resource(Edit, '/api/edit/') diff --git a/flask-consul/views/jms.py b/flask-consul/views/jms.py index 8a840c5..3510e19 100644 --- a/flask-consul/views/jms.py +++ b/flask-consul/views/jms.py @@ -45,19 +45,32 @@ class Jms(Resource): services_meta = consul_kv.get_services_meta(f'{vendor}_{account}_ecs').get('ecs_list', []) count_ecs = len(services_meta) count_off, count_on, count_cpu, count_mem, count_win, count_linux = 0, 0, 0, 0, 0, 0 + count_redis, count_mongodb, count_mysql, count_clickhouse = 0, 0, 0, 0 for i in services_meta: if i['os'] == 'linux': count_linux = count_linux + 1 + cpu = int(i['cpu'].replace('核', '')) + count_cpu = count_cpu + cpu + mem = float(i['mem'].replace('GB', '')) + count_mem = count_mem + mem elif i['os'] == 'windows': count_win = count_win + 1 + cpu = int(i['cpu'].replace('核', '')) + count_cpu = count_cpu + cpu + mem = float(i['mem'].replace('GB', '')) + count_mem = count_mem + mem + elif i['os'] in ['redis', 'redis6+']: + count_redis = count_redis + 1 + elif i['os'] == 'mysql': + count_mysql = count_mysql + 1 + elif i['os'] == 'mongodb': + count_mongodb = count_mongodb + 1 + elif i['os'] == 'clickhouse': + count_clickhouse = count_clickhouse + 1 if i.get('stat') == 'off': count_off = count_off + 1 else: count_on = count_on + 1 - cpu = int(i['cpu'].replace('核', '')) - count_cpu = count_cpu + cpu - mem = float(i['mem'].replace('GB', '')) - count_mem = count_mem + mem jms_job = consul_kv.get_value(f"ConsulManager/jms/jobs/{vendor}/{account}") if jms_job == {}: @@ -75,6 +88,10 @@ class Jms(Resource): 'account': account, 'count_linux': count_linux, 'count_win': count_win, + 'count_redis': count_redis, + 'count_mysql': count_mysql, + 'count_mongodb': count_mongodb, + 'count_clickhouse': count_clickhouse, 'count_mem': f'{count_mem}GB', 'count_cpu': f'{count_cpu}核', 'count_ecs': count_ecs, @@ -99,6 +116,15 @@ class Jms(Resource): linuxuid = ecs_info['linux'][-1] winport = ecs_info['windows'][0][0].split('/')[-1] winuid = ecs_info['windows'][-1] + redisport = ecs_info['redis'][0][0].split('/')[-1] + redisuid = ecs_info['redis'][-1] + mysqlport = ecs_info['mysql'][0][0].split('/')[-1] + mysqluid = ecs_info['mysql'][-1] + mongodbport = ecs_info['mongodb'][0][0].split('/')[-1] + mongodbuid = ecs_info['mongodb'][-1] + clickhouseport = ecs_info['clickhouse'][0][0].split('/')[-1] + clickhouseuid = ecs_info['clickhouse'][-1] + token = myaes.decrypt(jms_info['token']) custom_ecs_json = json.dumps(custom_ecs_info, indent=8) if custom_ecs_info != {} else '' jms_config = { @@ -109,6 +135,14 @@ class Jms(Resource): 'linuxuid': linuxuid, 'winport': winport, 'winuid': winuid, + 'redisport': redisport, + 'redisuid': redisuid, + 'mysqlport': mysqlport, + 'mysqluid': mysqluid, + 'mongodbport': mongodbport, + 'mongodbuid': mongodbuid, + 'clickhouseport': clickhouseport, + 'clickhouseuid': clickhouseuid, 'custom_ecs_info': custom_ecs_json, } else: @@ -125,6 +159,10 @@ class Jms(Resource): ecs_info = { "linux": [[f"ssh/{jms_config['linuxport']}"], jms_config['linuxuid']], "windows": [[f"rdp/{jms_config['winport']}"], jms_config['winuid']], + "redis": [[f"redis/{jms_config['redisport']}"], jms_config['redisuid']], + "mysql": [[f"mysql/{jms_config['mysqlport']}"], jms_config['mysqluid']], + "mongodb": [[f"mongodb/{jms_config['mongodbport']}"], jms_config['mongodbuid']], + "clickhouse": [[f"clickhouse/{jms_config['clickhouseport']}"], jms_config['clickhouseuid']], } consul_kv.put_kv('ConsulManager/jms/ecs_info', ecs_info) custom_ecs_info = jms_config['custom_ecs_info'] diff --git a/flask-consul/views/mongodb.py b/flask-consul/views/mongodb.py new file mode 100644 index 0000000..f486288 --- /dev/null +++ b/flask-consul/views/mongodb.py @@ -0,0 +1,93 @@ +from flask import Blueprint +from flask_restful import reqparse, Resource, Api +from flask_apscheduler import APScheduler +import traceback +#import sys +#sys.path.append("..") +from units import token_auth,consul_kv,gen_config,consul_svc +from units.config_log import * +blueprint = Blueprint('mongodb',__name__) +api = Api(blueprint) + +parser = reqparse.RequestParser() +parser.add_argument('job_id',type=str) +parser.add_argument('services_dict',type=dict) +parser.add_argument('cst_mongodb_dict',type=dict) +parser.add_argument('iid',type=str) +parser.add_argument('jobmongodb_name',type=str) +parser.add_argument('checked',type=str) + +class Mongodb(Resource): + decorators = [token_auth.auth.login_required] + def get(self, stype): + job_id = parser.parse_args()['job_id'] + if stype == 'jobmongodb': + jobmongodb = consul_kv.get_keys_list('ConsulManager/jobs') + jobmongodb_list = [i.split('/jobs/')[1] for i in jobmongodb if '/mongodb/' in i] + return {'code': 20000,'jobmongodb':jobmongodb_list} + elif stype == 'mongodb_services': + jobecs = consul_kv.get_keys_list('ConsulManager/jobs') + jobecs_list = [i.split('/jobs/')[1] for i in jobecs if '/mongodb/' in i] + services_list = [] + for i in jobecs_list: + serivces = i.split("/") + services_list.append(f'{serivces[0]}_{serivces[1]}_{serivces[2]}') + return {'code': 20000,'services_list': sorted(set(services_list))} + elif stype == 'mongodbrules': + return gen_config.get_mongodbrules() + elif stype == 'cstmongodbconf': + args = parser.parse_args() + iid = args['iid'] + cst_mongodb_config = consul_kv.get_value(f'ConsulManager/assets/sync_mongodb_custom/{iid}') + cst_mongodb_config.update({'iid': iid,'ipswitch': False,'portswitch': False}) + if 'ip' in cst_mongodb_config and cst_mongodb_config['ip'] != '': + cst_mongodb_config['ipswitch'] = True + if 'port' in cst_mongodb_config and cst_mongodb_config['port'] != '': + cst_mongodb_config['portswitch'] = True + return {'code': 20000, 'cst_mongodb': cst_mongodb_config} + elif stype == 'cstmongodblist': + args = parser.parse_args() + jobmongodb_name = args['jobmongodb_name'] + checked = args['checked'] + cst_mongodb_dict = consul_kv.get_kv_dict('ConsulManager/assets/sync_mongodb_custom/') + cst_mongodb_keylist = [k.split('/')[-1] for k,v in cst_mongodb_dict.items() if v != {}] + mongodb_info = consul_kv.get_res_services(jobmongodb_name) + if checked == 'false': + return mongodb_info + else: + cst_mongodb_list = [i for i in mongodb_info['res_list'] if i['iid'] in cst_mongodb_keylist] + return {'code': 20000, 'res_list': cst_mongodb_list} + + def post(self, stype): + if stype == 'mongodbpconfig': + args = parser.parse_args() + services_dict = args['services_dict'] + return gen_config.mongodb_config(services_dict['jobmongodb_list'],services_dict['cm_exporter'],services_dict['services_list'],services_dict['exporter']) + elif stype == 'cstmongodb': + args = parser.parse_args() + cst_mongodb_dict = args['cst_mongodb_dict'] + consul_mongodb_cst = {} + iid = cst_mongodb_dict['iid'] + try: + sid_dict = consul_svc.get_sid(iid)['instance'] + if cst_mongodb_dict['portswitch'] and cst_mongodb_dict['port'] != '': + consul_mongodb_cst['port'] = int(cst_mongodb_dict['port']) + sid_dict['Port'] = consul_mongodb_cst['port'] + if cst_mongodb_dict['ipswitch'] and cst_mongodb_dict['ip'] != '': + consul_mongodb_cst['ip'] = cst_mongodb_dict['ip'] + sid_dict['Address'] = consul_mongodb_cst['ip'] + consul_kv.put_kv(f'ConsulManager/assets/sync_mongodb_custom/{iid}',consul_mongodb_cst) + del sid_dict['Weights'] + del sid_dict['ContentHash'] + del sid_dict['Datacenter'] + sid_dict['name'] = sid_dict.pop('Service') + sid_dict['Meta']['instance'] = f"{sid_dict['Address']}:{sid_dict['Port']}" + sid_dict["check"] = { "tcp": sid_dict['Meta']['instance'],"interval": "60s" } + consul_svc.del_sid(iid) + consul_svc.add_sid(sid_dict) + return {'code': 20000, 'data': '自定义实例信息修改成功!'} + except Exception as e: + logger.error(f'{e}\n{traceback.format_exc()}') + return {'code': 50000, "data": '提交自定义实例信息格式错误!'} + +api.add_resource(Mongodb, '/api/mongodb/') diff --git a/flask-consul/views/polardb.py b/flask-consul/views/polardb.py new file mode 100644 index 0000000..e4c01a2 --- /dev/null +++ b/flask-consul/views/polardb.py @@ -0,0 +1,93 @@ +from flask import Blueprint +from flask_restful import reqparse, Resource, Api +from flask_apscheduler import APScheduler +import traceback +#import sys +#sys.path.append("..") +from units import token_auth,consul_kv,gen_config,consul_svc +from units.config_log import * +blueprint = Blueprint('polardb',__name__) +api = Api(blueprint) + +parser = reqparse.RequestParser() +parser.add_argument('job_id',type=str) +parser.add_argument('services_dict',type=dict) +parser.add_argument('cst_polardb_dict',type=dict) +parser.add_argument('iid',type=str) +parser.add_argument('jobpolardb_name',type=str) +parser.add_argument('checked',type=str) + +class Polardb(Resource): + decorators = [token_auth.auth.login_required] + def get(self, stype): + job_id = parser.parse_args()['job_id'] + if stype == 'jobpolardb': + jobpolardb = consul_kv.get_keys_list('ConsulManager/jobs') + jobpolardb_list = [i.split('/jobs/')[1] for i in jobpolardb if '/polardb/' in i] + return {'code': 20000,'jobpolardb':jobpolardb_list} + elif stype == 'polardb_services': + jobecs = consul_kv.get_keys_list('ConsulManager/jobs') + jobecs_list = [i.split('/jobs/')[1] for i in jobecs if '/polardb/' in i] + services_list = [] + for i in jobecs_list: + serivces = i.split("/") + services_list.append(f'{serivces[0]}_{serivces[1]}_{serivces[2]}') + return {'code': 20000,'services_list': sorted(set(services_list))} + elif stype == 'polardbrules': + return gen_config.get_polardbrules() + elif stype == 'cstpolardbconf': + args = parser.parse_args() + iid = args['iid'] + cst_polardb_config = consul_kv.get_value(f'ConsulManager/assets/sync_polardb_custom/{iid}') + cst_polardb_config.update({'iid': iid,'ipswitch': False,'portswitch': False}) + if 'ip' in cst_polardb_config and cst_polardb_config['ip'] != '': + cst_polardb_config['ipswitch'] = True + if 'port' in cst_polardb_config and cst_polardb_config['port'] != '': + cst_polardb_config['portswitch'] = True + return {'code': 20000, 'cst_polardb': cst_polardb_config} + elif stype == 'cstpolardblist': + args = parser.parse_args() + jobpolardb_name = args['jobpolardb_name'] + checked = args['checked'] + cst_polardb_dict = consul_kv.get_kv_dict('ConsulManager/assets/sync_polardb_custom/') + cst_polardb_keylist = [k.split('/')[-1] for k,v in cst_polardb_dict.items() if v != {}] + polardb_info = consul_kv.get_res_services(jobpolardb_name) + if checked == 'false': + return polardb_info + else: + cst_polardb_list = [i for i in polardb_info['res_list'] if i['iid'] in cst_polardb_keylist] + return {'code': 20000, 'res_list': cst_polardb_list} + + def post(self, stype): + if stype == 'polardbpconfig': + args = parser.parse_args() + services_dict = args['services_dict'] + return gen_config.polardb_config(services_dict['jobpolardb_list'],services_dict['cm_exporter'],services_dict['services_list'],services_dict['exporter']) + elif stype == 'cstpolardb': + args = parser.parse_args() + cst_polardb_dict = args['cst_polardb_dict'] + consul_polardb_cst = {} + iid = cst_polardb_dict['iid'] + try: + sid_dict = consul_svc.get_sid(iid)['instance'] + if cst_polardb_dict['portswitch'] and cst_polardb_dict['port'] != '': + consul_polardb_cst['port'] = int(cst_polardb_dict['port']) + sid_dict['Port'] = consul_polardb_cst['port'] + if cst_polardb_dict['ipswitch'] and cst_polardb_dict['ip'] != '': + consul_polardb_cst['ip'] = cst_polardb_dict['ip'] + sid_dict['Address'] = consul_polardb_cst['ip'] + consul_kv.put_kv(f'ConsulManager/assets/sync_polardb_custom/{iid}',consul_polardb_cst) + del sid_dict['Weights'] + del sid_dict['ContentHash'] + del sid_dict['Datacenter'] + sid_dict['name'] = sid_dict.pop('Service') + sid_dict['Meta']['instance'] = f"{sid_dict['Address']}:{sid_dict['Port']}" + sid_dict["check"] = { "tcp": sid_dict['Meta']['instance'],"interval": "60s" } + consul_svc.del_sid(iid) + consul_svc.add_sid(sid_dict) + return {'code': 20000, 'data': '自定义实例信息修改成功!'} + except Exception as e: + logger.error(f'{e}\n{traceback.format_exc()}') + return {'code': 50000, "data": '提交自定义实例信息格式错误!'} + +api.add_resource(Polardb, '/api/polardb/') diff --git a/flask-consul/views/selfclickhouse.py b/flask-consul/views/selfclickhouse.py new file mode 100644 index 0000000..91e74c0 --- /dev/null +++ b/flask-consul/views/selfclickhouse.py @@ -0,0 +1,80 @@ +import os +from flask import Blueprint +from flask_restful import reqparse, Resource, Api +import sys,traceback +sys.path.append("..") +from units import token_auth,selfclickhouse_manager +from werkzeug.datastructures import FileStorage +from units import upload +from units.config_log import * +blueprint = Blueprint('selfclickhouse',__name__) +api = Api(blueprint) + +parser = reqparse.RequestParser() +parser.add_argument('vendor',type=str) +parser.add_argument('account',type=str) +parser.add_argument('region',type=str) +parser.add_argument('group',type=str) +parser.add_argument('name',type=str) +parser.add_argument('ip',type=str) +parser.add_argument('port',type=str) +parser.add_argument('os',type=str) +parser.add_argument('del_dict',type=dict) +parser.add_argument('up_dict',type=dict) +parser.add_argument('file',type=FileStorage, location="files", help="File is wrong.") + +class Upload(Resource): + @token_auth.auth.login_required + def post(self): + file = parser.parse_args().get("file") + try: + filename = file.filename + file_extension = os.path.splitext(filename)[1].lower() + try: + file_data = file.read() + if file_extension == '.xlsx': + return upload.read_execl(file_data,'selfclickhouse') + elif file_extension == '.csv': + return upload.read_csv(file_data,'selfclickhouse') + except Exception as e: + logger.error(f"【selfclickhouse】文件后缀名错误,请导入xlsx或csv格式,{e}\n{traceback.format_exc()}") + return {"code": 50000, "data": f"文件后缀名错误,请导入xlsx或csv格式!"} + except Exception as e: + logger.error(f"【selfclickhouse】导入失败,{e}\n{traceback.format_exc()}") + return {"code": 50000, "data": f"导入失败!"} + +class GetAllList(Resource): + @token_auth.auth.login_required + def get(self): + args = parser.parse_args() + return selfclickhouse_manager.get_all_list(args['vendor'],args['account'],args['region'],args['group']) + +class SelfclickhouseApi(Resource): + decorators = [token_auth.auth.login_required] + def get(self): + return selfclickhouse_manager.get_service() + def post(self): + args = parser.parse_args() + logger.info(f'=======\n{args}') + return selfclickhouse_manager.add_service(args['vendor'],args['account'],args['region'], + args['group'],args['name'],args['ip'],args['port'],args['os']) + def put(self): + args = parser.parse_args() + del_dict = args['del_dict'] + up_dict = args['up_dict'] + resp_del = selfclickhouse_manager.del_service(del_dict['vendor'],del_dict['account'], + del_dict['region'],del_dict['group'],del_dict['name']) + resp_add = selfclickhouse_manager.add_service(up_dict['vendor'],up_dict['account'],up_dict['region'], + up_dict['group'],up_dict['name'],up_dict['ip'], + up_dict['port'],up_dict['os']) + if resp_del["code"] == 20000 and resp_add["code"] == 20000: + return {"code": 20000, "data": f"更新成功!"} + else: + return {"code": 50000, "data": f"更新失败!"} + def delete(self): + args = parser.parse_args() + return selfclickhouse_manager.del_service(args['vendor'],args['account'],args['region'],args['group'],args['name']) + +api.add_resource(GetAllList,'/api/selfclickhouse/alllist') +api.add_resource(SelfclickhouseApi, '/api/selfclickhouse/service') +api.add_resource(Upload,'/api/selfclickhouse/upload') diff --git a/flask-consul/views/selfmongodb.py b/flask-consul/views/selfmongodb.py new file mode 100644 index 0000000..55f8b5f --- /dev/null +++ b/flask-consul/views/selfmongodb.py @@ -0,0 +1,80 @@ +import os +from flask import Blueprint +from flask_restful import reqparse, Resource, Api +import sys,traceback +sys.path.append("..") +from units import token_auth,selfmongodb_manager +from werkzeug.datastructures import FileStorage +from units import upload +from units.config_log import * +blueprint = Blueprint('selfmongodb',__name__) +api = Api(blueprint) + +parser = reqparse.RequestParser() +parser.add_argument('vendor',type=str) +parser.add_argument('account',type=str) +parser.add_argument('region',type=str) +parser.add_argument('group',type=str) +parser.add_argument('name',type=str) +parser.add_argument('ip',type=str) +parser.add_argument('port',type=str) +parser.add_argument('os',type=str) +parser.add_argument('del_dict',type=dict) +parser.add_argument('up_dict',type=dict) +parser.add_argument('file',type=FileStorage, location="files", help="File is wrong.") + +class Upload(Resource): + @token_auth.auth.login_required + def post(self): + file = parser.parse_args().get("file") + try: + filename = file.filename + file_extension = os.path.splitext(filename)[1].lower() + try: + file_data = file.read() + if file_extension == '.xlsx': + return upload.read_execl(file_data,'selfmongodb') + elif file_extension == '.csv': + return upload.read_csv(file_data,'selfmongodb') + except Exception as e: + logger.error(f"【selfmongodb】文件后缀名错误,请导入xlsx或csv格式,{e}\n{traceback.format_exc()}") + return {"code": 50000, "data": f"文件后缀名错误,请导入xlsx或csv格式!"} + except Exception as e: + logger.error(f"【selfmongodb】导入失败,{e}\n{traceback.format_exc()}") + return {"code": 50000, "data": f"导入失败!"} + +class GetAllList(Resource): + @token_auth.auth.login_required + def get(self): + args = parser.parse_args() + return selfmongodb_manager.get_all_list(args['vendor'],args['account'],args['region'],args['group']) + +class SelfmongodbApi(Resource): + decorators = [token_auth.auth.login_required] + def get(self): + return selfmongodb_manager.get_service() + def post(self): + args = parser.parse_args() + logger.info(f'=======\n{args}') + return selfmongodb_manager.add_service(args['vendor'],args['account'],args['region'], + args['group'],args['name'],args['ip'],args['port'],args['os']) + def put(self): + args = parser.parse_args() + del_dict = args['del_dict'] + up_dict = args['up_dict'] + resp_del = selfmongodb_manager.del_service(del_dict['vendor'],del_dict['account'], + del_dict['region'],del_dict['group'],del_dict['name']) + resp_add = selfmongodb_manager.add_service(up_dict['vendor'],up_dict['account'],up_dict['region'], + up_dict['group'],up_dict['name'],up_dict['ip'], + up_dict['port'],up_dict['os']) + if resp_del["code"] == 20000 and resp_add["code"] == 20000: + return {"code": 20000, "data": f"更新成功!"} + else: + return {"code": 50000, "data": f"更新失败!"} + def delete(self): + args = parser.parse_args() + return selfmongodb_manager.del_service(args['vendor'],args['account'],args['region'],args['group'],args['name']) + +api.add_resource(GetAllList,'/api/selfmongodb/alllist') +api.add_resource(SelfmongodbApi, '/api/selfmongodb/service') +api.add_resource(Upload,'/api/selfmongodb/upload') diff --git a/flask-consul/views/selfpolardb.py b/flask-consul/views/selfpolardb.py new file mode 100644 index 0000000..957fe6c --- /dev/null +++ b/flask-consul/views/selfpolardb.py @@ -0,0 +1,80 @@ +import os +from flask import Blueprint +from flask_restful import reqparse, Resource, Api +import sys,traceback +sys.path.append("..") +from units import token_auth,selfpolardb_manager +from werkzeug.datastructures import FileStorage +from units import upload +from units.config_log import * +blueprint = Blueprint('selfpolardb',__name__) +api = Api(blueprint) + +parser = reqparse.RequestParser() +parser.add_argument('vendor',type=str) +parser.add_argument('account',type=str) +parser.add_argument('region',type=str) +parser.add_argument('group',type=str) +parser.add_argument('name',type=str) +parser.add_argument('ip',type=str) +parser.add_argument('port',type=str) +parser.add_argument('os',type=str) +parser.add_argument('del_dict',type=dict) +parser.add_argument('up_dict',type=dict) +parser.add_argument('file',type=FileStorage, location="files", help="File is wrong.") + +class Upload(Resource): + @token_auth.auth.login_required + def post(self): + file = parser.parse_args().get("file") + try: + filename = file.filename + file_extension = os.path.splitext(filename)[1].lower() + try: + file_data = file.read() + if file_extension == '.xlsx': + return upload.read_execl(file_data,'selfpolardb') + elif file_extension == '.csv': + return upload.read_csv(file_data,'selfpolardb') + except Exception as e: + logger.error(f"【selfpolardb】文件后缀名错误,请导入xlsx或csv格式,{e}\n{traceback.format_exc()}") + return {"code": 50000, "data": f"文件后缀名错误,请导入xlsx或csv格式!"} + except Exception as e: + logger.error(f"【selfpolardb】导入失败,{e}\n{traceback.format_exc()}") + return {"code": 50000, "data": f"导入失败!"} + +class GetAllList(Resource): + @token_auth.auth.login_required + def get(self): + args = parser.parse_args() + return selfpolardb_manager.get_all_list(args['vendor'],args['account'],args['region'],args['group']) + +class SelfpolardbApi(Resource): + decorators = [token_auth.auth.login_required] + def get(self): + return selfpolardb_manager.get_service() + def post(self): + args = parser.parse_args() + logger.info(f'=======\n{args}') + return selfpolardb_manager.add_service(args['vendor'],args['account'],args['region'], + args['group'],args['name'],args['ip'],args['port'],args['os']) + def put(self): + args = parser.parse_args() + del_dict = args['del_dict'] + up_dict = args['up_dict'] + resp_del = selfpolardb_manager.del_service(del_dict['vendor'],del_dict['account'], + del_dict['region'],del_dict['group'],del_dict['name']) + resp_add = selfpolardb_manager.add_service(up_dict['vendor'],up_dict['account'],up_dict['region'], + up_dict['group'],up_dict['name'],up_dict['ip'], + up_dict['port'],up_dict['os']) + if resp_del["code"] == 20000 and resp_add["code"] == 20000: + return {"code": 20000, "data": f"更新成功!"} + else: + return {"code": 50000, "data": f"更新失败!"} + def delete(self): + args = parser.parse_args() + return selfpolardb_manager.del_service(args['vendor'],args['account'],args['region'],args['group'],args['name']) + +api.add_resource(GetAllList,'/api/selfpolardb/alllist') +api.add_resource(SelfpolardbApi, '/api/selfpolardb/service') +api.add_resource(Upload,'/api/selfpolardb/upload') diff --git a/tools/del_consul_svc.sh b/tools/del_consul_svc.sh new file mode 100644 index 0000000..1bb56df --- /dev/null +++ b/tools/del_consul_svc.sh @@ -0,0 +1,12 @@ +# Assuming localhost, substitute with actual Consul server address. +export CONSUL_HTTP_TOKEN=234dbcac-25df-42d3-965a-af4193474a56 +CONSUL_HOST="http://172.26.32.83:8500" + +# Get all instances of the service +SERVICE_INSTANCES=$(curl -s -H "X-Consul-Token: $CONSUL_HTTP_TOKEN" "${CONSUL_HOST}/v1/catalog/service/alicloud_dreame_app_mongodb") + +# Extract the service IDs and deregister each +echo "${SERVICE_INSTANCES}" | jq -r '.[].ServiceID' | while read SERVICE_ID; do + echo "Deregistering service instance: $SERVICE_ID" + curl -s -X PUT -H "X-Consul-Token: $CONSUL_HTTP_TOKEN" "${CONSUL_HOST}/v1/agent/service/deregister/${SERVICE_ID}" +done \ No newline at end of file diff --git a/vue-consul/src/views/jms/index.vue b/vue-consul/src/views/jms/index.vue index bf92309..46e243f 100644 --- a/vue-consul/src/views/jms/index.vue +++ b/vue-consul/src/views/jms/index.vue @@ -57,6 +57,26 @@ rdp端口:   管理用户ID: +
+

Redis:

+ redis端口: +   管理用户ID: +
+
+

Mongodb:

+ mongodb端口: +   管理用户ID: +
+
+

Clickhouse:

+ clickhouse端口: +   管理用户ID: +
+
+

Mysql:

+ mysql端口: +   管理用户ID: +

全局特殊主机【管理用户】信息:

+ + + + @@ -130,7 +154,9 @@ import { getJmsList, getJmsConfig, postJmsConfig, postJmsSwitch, postJmsSync } f export default { data() { return { - jms_config: { ver: 'V2', url: '', token: '', linuxport: '22', linuxuid: '', winport: '3389', winuid: '', custom_ecs_info: '' }, + jms_config: { ver: 'V2', url: '', token: '', linuxport: '22', linuxuid: '', winport: '3389', winuid: '', + redisport: '6379', redisuid: '', mysqlport: '3306', mysqluid: '', mongodbport: '3717', mongodbuid: '', + clickhouseport: '9000', clickhouseuid: '', custom_ecs_info: '' }, listLoading: false, dialogFormVisible: false, query: { vendor: '', account: '' }, diff --git a/vue-consul/src/views/node-exporter/jobs.vue b/vue-consul/src/views/node-exporter/jobs.vue index a36f46f..ade3adc 100644 --- a/vue-consul/src/views/node-exporter/jobs.vue +++ b/vue-consul/src/views/node-exporter/jobs.vue @@ -144,6 +144,9 @@ ECS MySQL REDIS + POLARDB + MONGODB + CLICKHOUSE @@ -170,6 +173,15 @@ + + + + + + + + +