From 0baa27bcd89a04dfcef5f43b65e89e59f46019a8 Mon Sep 17 00:00:00 2001 From: "StarsL.cn" Date: Sun, 17 Jul 2022 22:38:23 +0800 Subject: [PATCH] =?UTF-8?q?=E4=BA=91=E4=B8=BB=E6=9C=BA=E5=90=8C=E6=AD=A5ju?= =?UTF-8?q?mpserver=E5=8A=9F=E8=83=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../workflows/{docker-image.yml => main.yml} | 0 flask-consul/manager.py | 12 +- flask-consul/units/cloud/alicloud.py | 119 +++++++------- flask-consul/units/jms/sync_jms.py | 95 +++++++++++ flask-consul/views/exp.py | 1 - flask-consul/views/jms.py | 86 ++++++++++ vue-consul/src/api/jms.js | 31 ++++ vue-consul/src/router/index.js | 8 +- vue-consul/src/views/jms/index.vue | 147 ++++++++++++++++++ 9 files changed, 440 insertions(+), 59 deletions(-) rename .github/workflows/{docker-image.yml => main.yml} (100%) create mode 100755 flask-consul/units/jms/sync_jms.py create mode 100644 flask-consul/views/jms.py create mode 100644 vue-consul/src/api/jms.js create mode 100644 vue-consul/src/views/jms/index.vue diff --git a/.github/workflows/docker-image.yml b/.github/workflows/main.yml similarity index 100% rename from .github/workflows/docker-image.yml rename to .github/workflows/main.yml diff --git a/flask-consul/manager.py b/flask-consul/manager.py index a0fcdec..1db8333 100755 --- a/flask-consul/manager.py +++ b/flask-consul/manager.py @@ -2,13 +2,16 @@ from flask import Flask from units import consul_kv import uuid + skey_path = 'ConsulManager/assets/secret/skey' if consul_kv.get_kv_dict(skey_path) == {}: consul_kv.put_kv(skey_path,{'sk':''.join(str(uuid.uuid4()).split('-'))}) -from views import login, blackbox, consul, jobs, nodes, selfnode, avd, exp +from views import login, blackbox, consul, jobs, nodes, selfnode, avd, exp, jms from units.cloud import huaweicloud,alicloud,tencent_cloud from units.avd import avd_list +from units.jms import sync_jms + app = Flask(__name__) app.register_blueprint(login.blueprint) app.register_blueprint(blackbox.blueprint) @@ -18,18 +21,23 @@ app.register_blueprint(nodes.blueprint) app.register_blueprint(selfnode.blueprint) app.register_blueprint(avd.blueprint) app.register_blueprint(exp.blueprint) +app.register_blueprint(jms.blueprint) + class Config(object): JOBS = [] SCHEDULER_API_ENABLED = True + ecs_jobs = consul_kv.get_kv_dict('ConsulManager/jobs') avd_jobs = consul_kv.get_kv_dict('ConsulManager/avd/jobs') exp_jobs = consul_kv.get_kv_dict('ConsulManager/exp/jobs') -init_jobs = { **ecs_jobs, **avd_jobs, **exp_jobs } +jms_jobs = consul_kv.get_kv_dict('ConsulManager/jms/jobs') +init_jobs = { **ecs_jobs, **avd_jobs, **exp_jobs, **jms_jobs } if init_jobs is not None: for k,v in init_jobs.items(): print(f'【初始化任务】{k}:\n {v}', flush=True) Config.JOBS = init_jobs.values() + app.config.from_object(Config()) if __name__ == "__main__": diff --git a/flask-consul/units/cloud/alicloud.py b/flask-consul/units/cloud/alicloud.py index cfb9b69..85038db 100644 --- a/flask-consul/units/cloud/alicloud.py +++ b/flask-consul/units/cloud/alicloud.py @@ -15,72 +15,81 @@ from units.cloud import sync_ecs from units.cloud import notify def exp(account,collect_days,notify_days,notify_amount): + print(f"=====【阿里云：余额与到期日统计开始：{account}】", flush=True) ak,sk = consul_kv.get_aksk('alicloud',account) now = datetime.datetime.utcnow().strftime('%Y-%m-%dT16:00:00Z') collect = (datetime.datetime.utcnow() + datetime.timedelta(days=collect_days+1)).strftime('%Y-%m-%dT16:00:00Z') config = open_api_models.Config(access_key_id=ak,access_key_secret=sk) config.endpoint = f'business.aliyuncs.com' client = BssOpenApi20171214Client(config) + try: + amount_response = client.query_account_balance() + if amount_response.body.success: + available_amount = amount_response.body.data.available_amount + amount = float(available_amount.replace(',','')) + consul_kv.put_kv(f'ConsulManager/exp/lists/alicloud/{account}/amount',{'amount':amount}) + print('alicloud',account,f'可用余额:{available_amount}', flush=True) + amount_dict = {} + if amount < notify_amount: + amount_dict = {'amount':amount} + content = f'### 阿里云账号 {account}：\n### 可用余额：{amount} 元' + if exp_config['switch'] and exp_config.get('wecom',False): + notify.wecom(wecomwh,content) + if exp_config['switch'] and exp_config.get('dingding',False): + notify.dingding(dingdingwh,content) + if exp_config['switch'] and exp_config.get('feishu',False): + title = '阿里云余额不足通知' + md = content + notify.feishu(feishuwh,title,md) + else: + print(f'查询失败，Code:{amount_response.body.code}, 信息:{amount_response.body.message}, requestId:{amount_response.body.request_id}', flush=True) + except Exception as e: + print('==ERROR==',e,flush=True) + raise query_available_instances_request = bss_open_api_20171214_models.QueryAvailableInstancesRequest( - renew_status='ManualRenewal', - end_time_start=now, - end_time_end=collect) + renew_status='ManualRenewal',end_time_start=now,end_time_end=collect) runtime = util_models.RuntimeOptions() - amount_response = client.query_account_balance() try: exp = client.query_available_instances_with_options(query_available_instances_request, runtime) exp_list = exp.body.to_map()['Data']['InstanceList'] - exp_dict = {} - isnotify_list = consul_kv.get_keys_list(f'ConsulManager/exp/isnotify/alicloud/{account}') - isnotify_list = [i.split('/')[-1] for i in isnotify_list] - notify_dict = {} - amount_dict = {} - for i in exp_list: - notify_id = hashlib.md5(str(i).encode(encoding='UTF-8')).hexdigest() - endtime = datetime.datetime.strptime(i['EndTime'],'%Y-%m-%dT%H:%M:%SZ') + datetime.timedelta(hours=8) - endtime_str = endtime.strftime('%Y-%m-%d') - iname = consul_svc.get_sid(i['InstanceID'])['instance']['Meta']['name'] if i['ProductCode'] == 'ecs' else 'Null' - exp_dict[i['InstanceID']] = {'Region':i.get('Region','Null'),'Product':i['ProductCode'], - 'Name':iname,'EndTime':endtime_str,'notify_id':notify_id, - 'Ptype':i.get('ProductType',i['ProductCode'])} - if (endtime - datetime.datetime.now()).days < notify_days and notify_id not in isnotify_list: - notify_dict[i['InstanceID']] = exp_dict[i['InstanceID']] - consul_kv.put_kv(f'ConsulManager/exp/lists/alicloud/{account}/exp', exp_dict) - amount = float(amount_response.body.data.available_amount.replace(',','')) - consul_kv.put_kv(f'ConsulManager/exp/lists/alicloud/{account}/amount',{'amount':amount}) - if amount < notify_amount: - amount_dict = {'amount':amount} - exp_config = consul_kv.get_value('ConsulManager/exp/config') - wecomwh = exp_config.get('wecomwh','') - dingdingwh = exp_config.get('dingdingwh','') - feishuwh = exp_config.get('feishuwh','') - if notify_dict != {}: - msg = [f'### 阿里云账号 {account}：\n### 以下资源到期日小于 {notify_days} 天：'] - for k,v in notify_dict.items(): - iname = k if v['Name'] == 'Null' else v['Name'] - msg.append(f"- {v['Region']}：{v['Product']}：{iname}：{v['EndTime']}") - content = '\n'.join(msg) - if exp_config['switch'] and exp_config.get('wecom',False): - notify.wecom(wecomwh,content) - if exp_config['switch'] and exp_config.get('dingding',False): - notify.dingding(dingdingwh,content) - if exp_config['switch'] and exp_config.get('feishu',False): - title = '阿里云资源到期通知' - md = content - notify.feishu(feishuwh,title,md) - if amount_dict != {}: - content = f'### 阿里云账号 {account}：\n### 可用余额：{amount} 元' - if exp_config['switch'] and exp_config.get('wecom',False): - notify.wecom(wecomwh,content) - if exp_config['switch'] and exp_config.get('dingding',False): - notify.dingding(dingdingwh,content) - if exp_config['switch'] and exp_config.get('feishu',False): - title = '阿里云余额不足通知' - md = content - notify.feishu(feishuwh,title,md) - - except Exception as error: - UtilClient.assert_as_string(error.message) + except Exception as e: + #exp_list = [] + print('==ERROR==',e,flush=True) + raise + exp_dict = {} + isnotify_list = consul_kv.get_keys_list(f'ConsulManager/exp/isnotify/alicloud/{account}') + isnotify_list = [i.split('/')[-1] for i in isnotify_list] + notify_dict = {} + for i in exp_list: + notify_id = hashlib.md5(str(i).encode(encoding='UTF-8')).hexdigest() + endtime = datetime.datetime.strptime(i['EndTime'],'%Y-%m-%dT%H:%M:%SZ') + datetime.timedelta(hours=8) + endtime_str = endtime.strftime('%Y-%m-%d') + iname = consul_svc.get_sid(i['InstanceID'])['instance']['Meta']['name'] if i['ProductCode'] == 'ecs' else 'Null' + exp_dict[i['InstanceID']] = {'Region':i.get('Region','Null'),'Product':i['ProductCode'], + 'Name':iname,'EndTime':endtime_str,'notify_id':notify_id, + 'Ptype':i.get('ProductType',i['ProductCode'])} + if (endtime - datetime.datetime.now()).days < notify_days and notify_id not in isnotify_list: + notify_dict[i['InstanceID']] = exp_dict[i['InstanceID']] + consul_kv.put_kv(f'ConsulManager/exp/lists/alicloud/{account}/exp', exp_dict) + exp_config = consul_kv.get_value('ConsulManager/exp/config') + wecomwh = exp_config.get('wecomwh','') + dingdingwh = exp_config.get('dingdingwh','') + feishuwh = exp_config.get('feishuwh','') + if notify_dict != {}: + msg = [f'### 阿里云账号 {account}：\n### 以下资源到期日小于 {notify_days} 天：'] + for k,v in notify_dict.items(): + iname = k if v['Name'] == 'Null' else v['Name'] + msg.append(f"- {v['Region']}：{v['Product']}：{iname}：{v['EndTime']}") + content = '\n'.join(msg) + if exp_config['switch'] and exp_config.get('wecom',False): + notify.wecom(wecomwh,content) + if exp_config['switch'] and exp_config.get('dingding',False): + notify.dingding(dingdingwh,content) + if exp_config['switch'] and exp_config.get('feishu',False): + title = '阿里云资源到期通知' + md = content + notify.feishu(feishuwh,title,md) + print(f"=====【阿里云：余额与到期日统计结束：{account}】", flush=True) def group(account): ak,sk = consul_kv.get_aksk('alicloud',account) diff --git a/flask-consul/units/jms/sync_jms.py b/flask-consul/units/jms/sync_jms.py new file mode 100755 index 0000000..9dd8c69 --- /dev/null +++ b/flask-consul/units/jms/sync_jms.py @@ -0,0 +1,95 @@ +import datetime,requests,json +from units import consul_kv,consul_manager,myaes + +jms = consul_kv.get_value('ConsulManager/jms/jms_info') +jms_url = jms.get('url') +token = myaes.decrypt(jms.get('token')) +headers = {'Content-Type': 'application/json','Authorization': f"Token {token}"} + +#创建node +def create_node(node_id,cloud,account): + node_url = f"{jms_url}/api/v1/assets/nodes/{node_id}/children/" + jms_node_list = requests.request("GET", node_url, headers=headers).json() + if type(jms_node_list) == dict: + print(' 【JMS】',jms_node_list.get('detail'),flush=True) + cloud_group_dict = consul_kv.get_value(f'ConsulManager/assets/{cloud}/group/{account}') + for k,v in cloud_group_dict.items(): + if v not in [i['value'] for i in jms_node_list]: + response = requests.request("POST", node_url, headers=headers, data = json.dumps({'value': v})) + print(' 【JMS】新增组===>',v,response.status_code,flush=True) + reget_node_list = requests.request("GET", node_url, headers=headers).json() + new_node_dict = {i['value']:i['id'] for i in reget_node_list} + return new_node_dict + +def update_jms_ecs(new_node_dict,node_id,cloud,account,ecs_info,custom_ecs_info): + #比较云主机与JMS中对应node的主机列表，删除jms中多余的主机 + ecs_url = f"{jms_url}/api/v1/assets/assets/" + reget_ecs_list = requests.request("GET", f'{ecs_url}?node={node_id}', headers=headers).json() + jms_ecs_dict = {i['ip']:{'name':i['hostname'],'id':i['id'],'comment':i['comment'],'node':i['nodes_display'][0]} for i in reget_ecs_list} + ecs_list = consul_manager.get_instances(f'{cloud}_{account}_ecs')['instances'] + ecs_dict = {i['ID']:{'name':i['meta'][0]['name'],'ip':i['address'],'ent':i['meta'][0]['group'],'ostype':i['meta'][0]['os'],'region':i['meta'][0]['region'],'vendor':i['meta'][0]['vendor']} for i in ecs_list} + del_ecs_list = [v['id'] for k,v in jms_ecs_dict.items() if k not in [i['ip'] for i in ecs_dict.values()]] + for del_ecs in del_ecs_list: + response = requests.request("DELETE", f'{ecs_url}{del_ecs}/', headers=headers) + print(' 【JMS】删除主机：',del_ecs,response.status_code,flush=True) + + #增加/更新缺少的主机 + for k,v in ecs_dict.items(): + ip = v['ip'] + iname = v['name'] + nodes = new_node_dict[v['ent']] + ostype = v['ostype'] + comment = f"{v['vendor']} {account} {v['region']} {k}" + protocols = ecs_info[ostype][0] + platform = ostype.title() + admin_user = ecs_info[ostype][1] + if custom_ecs_info != {}: + if len([i for i in custom_ecs_info.keys() if i.lower() in iname.lower()]) > 0: + for custom_name,custom_info in custom_ecs_info.items(): + if custom_name.lower() in iname.lower(): + protocols = custom_info[ostype][0] + platform = ostype.title() + admin_user = custom_info[ostype][1] + payload = { + "ip": ip, + "hostname": "cm_" + iname, + "protocols": protocols, + "platform": platform, + "is_active": True, + "domain": "", + "admin_user": admin_user, + "nodes": [nodes], + "comment": comment + } + if ip in jms_ecs_dict.keys(): + if jms_ecs_dict[ip]['name'] != "cm_" + iname or jms_ecs_dict[ip]['node'].split('/')[-1] != v['ent']: + response = requests.request("PUT", f"{ecs_url}{jms_ecs_dict[ip]['id']}/", headers=headers, data = json.dumps(payload)) + print(' 【JMS】update：主机名：',response.json()['hostname'],response.status_code,flush=True) + else: + response = requests.request("POST", ecs_url, headers=headers, data = json.dumps(payload)) + print(' 【JMS】add：主机名：',response.json()['hostname'],response.status_code,flush=True) + return 'ok' + +#从JMS中删除没有主机的组 +def del_node(node_id,cloud,account): + node_tree_url = f"{jms_url}/api/v1/assets/nodes/children/tree/?id={node_id}" + jms_node_list = requests.request("GET", node_tree_url, headers=headers).json() + for i in jms_node_list: + if i['name'].endswith(' (0)'): + del_node_url = f"{jms_url}/api/v1/assets/nodes/{i['meta']['node']['id']}/" + response = requests.request("DELETE", del_node_url, headers=headers) + print(' 【JMS】删除空组===>',i['name'],response.status_code,flush=True) + return 'ok' + +def run(cloud,account): + print('【JOB】===>',cloud,account,'JMS同步开始',flush=True) + node_id = consul_kv.get_value(f'ConsulManager/jms/{cloud}/{account}/node_id')['node_id'] + temp_ecs_info = consul_kv.get_value(f'ConsulManager/jms/{cloud}/{account}/ecs_info') + ecs_info = consul_kv.get_value(f'ConsulManager/jms/ecs_info') if temp_ecs_info == {} else temp_ecs_info + temp_custom_ecs_info = consul_kv.get_value(f'ConsulManager/jms/{cloud}/{account}/custom_ecs_info') + custom_ecs_info = consul_kv.get_value(f'ConsulManager/jms/custom_ecs_info') if temp_custom_ecs_info == {} else temp_custom_ecs_info + + new_node_dict = create_node(node_id,cloud,account) + update_jms_ecs(new_node_dict,node_id,cloud,account,ecs_info,custom_ecs_info) + del_node(node_id,cloud,account) + print('【JOB】===>',cloud,account,'JMS同步完成',flush=True) diff --git a/flask-consul/views/exp.py b/flask-consul/views/exp.py index c5bcc6b..de1b884 100644 --- a/flask-consul/views/exp.py +++ b/flask-consul/views/exp.py @@ -27,7 +27,6 @@ class Exp(Resource): query_set = set({k:v for k,v in query_dict.items() if v != ''}.items()) cloud_job_list = consul_kv.get_keys_list('ConsulManager/jobs') cloud_list = [i for i in cloud_job_list if i.endswith('/group')] - exp_dict = consul_kv.get_kv_dict(f'ConsulManager/exp/lists') exp_list = [] amount_list = [] for i in cloud_list: diff --git a/flask-consul/views/jms.py b/flask-consul/views/jms.py new file mode 100644 index 0000000..5fdf323 --- /dev/null +++ b/flask-consul/views/jms.py @@ -0,0 +1,86 @@ +from flask import Blueprint +from flask_restful import reqparse, Resource, Api +from flask_apscheduler import APScheduler +from units import token_auth,consul_kv,myaes +from config import vendors +import json +from .jobs import deljob,addjob,runjob +blueprint = Blueprint('jms',__name__) +api = Api(blueprint) + +parser = reqparse.RequestParser() +parser.add_argument('query_dict',type=str) +parser.add_argument('jms_config',type=dict) +parser.add_argument('isnotify_dict',type=dict) + +class Jms(Resource): + decorators = [token_auth.auth.login_required] + def get(self,stype): + if stype == 'list': + switch = consul_kv.get_value(f'ConsulManager/jms/jms_info') + if switch == {}: + return({'code': 20000,'exp_list':[],'vendor_list':[],'account_list':[]}) + args = parser.parse_args() + query_dict = json.loads(args['query_dict']) + if query_dict['vendor'] != '': + query_dict['vendor'] = {v : k for k, v in vendors.items()}[query_dict['vendor']] + query_set = set({k:v for k,v in query_dict.items() if v != ''}.items()) + cloud_job_list = consul_kv.get_keys_list('ConsulManager/jobs') + cloud_list = [i for i in cloud_job_list if i.endswith('/group')] + + exp_list = [] + for i in cloud_list: + vendor,account = i.split('/')[2:4] + cloud_info_dict = {'vendor':vendor,'account':account} + if query_set.issubset(cloud_info_dict.items()): + pass + else: + continue + count_group = consul_kv.get_value(f'ConsulManager/record/jobs/{vendor}/{account}/group')['count'] + services_meta = consul_kv.get_services_meta(f'{vendor}_{account}_ecs').get('ecs_list',[]) + count_ecs = len(services_meta) + count_cpu,count_mem,count_win,count_linux = 0,0,0,0 + for i in services_meta: + if i['os'] == linux: + count_linux = count_linux + 1 + elif i['os'] == windows: + count_win = count_win + 1 + cpu = int(i['cpu'].replace('核','')) + count_cpu = count_cpu + cpu + mem = int(i['cpu'].replace('GB','')) + count_mem = count_mem + mem + + exp_list.append({'vendor':vendors[vendor],'account':account,'id':k,'Region':v['Region'], + 'Product':v['Product'],'Name':v.get('Name','Null'),'EndTime':v['EndTime'], + 'Ptype':v['Ptype'].replace('hws.resource.type.',''), + 'notify_id': v['notify_id'],'isnotify':isnotify}) + vendor_list = sorted(list(set([i['vendor'] for i in exp_list]))) + account_list = sorted(list(set([i['account'] for i in exp_list]))) + return {'code': 20000,'exp_list':exp_list,'vendor_list':vendor_list,'account_list':account_list,'amount_list':amount_list} + if stype == 'config': + ecs_info = consul_kv.get_value('ConsulManager/jms/ecs_info') + jms_info = consul_kv.get_value('ConsulManager/jms/jms_info') + if ecs_info != {} and jms_info != {}: + linuxport = ecs_info['linux'][0][0].split('/')[-1] + linuxuid = ecs_info['linux'][-1] + winport = ecs_info['windows'][0][0].split('/')[-1] + winuid = ecs_info['windows'][-1] + token = myaes.decrypt(jms_info['token']) + jms_config = {'url': jms_info['url'], 'token': token, + 'linuxport': linuxport, 'linuxuid': linuxuid, + 'winport': winport, 'winuid': winuid} + else: + jms_config = {} + return {'code': 20000, 'jms_config': jms_config} + def post(self,stype): + if stype == 'config': + args = parser.parse_args() + jms_config = args['jms_config'] + token = myaes.encrypt(jms_config['token']) + jms_info = {'url': jms_config['url'], 'token': token} + consul_kv.put_kv('ConsulManager/jms/jms_info', jms_info) + ecs_info = {"linux": [[f"ssh/{jms_config['linuxport']}"],jms_config['linuxuid']], + "windows": [[f"rdp/{jms_config['winport']}"],jms_config['winuid']]} + consul_kv.put_kv('ConsulManager/jms/ecs_info', ecs_info) + return {'code': 20000, 'data': '配置完成'} +api.add_resource(Jms, '/api/jms/') diff --git a/vue-consul/src/api/jms.js b/vue-consul/src/api/jms.js new file mode 100644 index 0000000..b58f65c --- /dev/null +++ b/vue-consul/src/api/jms.js @@ -0,0 +1,31 @@ +import request from '@/utils/request-ops' + +export function getExpList(query_dict) { + return request({ + url: '/api/exp/list', + method: 'get', + params: { query_dict } + }) +} +export function getJmsConfig() { + return request({ + url: '/api/jms/config', + method: 'get' + }) +} + +export function postJmsConfig(jms_config) { + return request({ + url: '/api/jms/config', + method: 'post', + data: { jms_config } + }) +} + +export function postExpIsnotify(isnotify_dict) { + return request({ + url: '/api/exp/isnotify', + method: 'post', + data: { isnotify_dict } + }) +} diff --git a/vue-consul/src/router/index.js b/vue-consul/src/router/index.js index 1b6fc7c..3ce1774 100644 --- a/vue-consul/src/router/index.js +++ b/vue-consul/src/router/index.js @@ -87,7 +87,7 @@ export const constantRoutes = [ component: Layout, redirect: '/nodes/jobs', name: '云资源监控', - meta: { title: '云资源监控', icon: 'example' }, + meta: { title: '云资源监控', icon: 'el-icon-shopping-bag-2' }, children: [ { path: 'jobs', @@ -95,6 +95,12 @@ export const constantRoutes = [ component: () => import('@/views/node-exporter/jobs'), meta: { title: '接入数据源', icon: 'el-icon-school' } }, + { + path: 'jms', + name: 'JumpServer', + component: () => import('@/views/jms/index'), + meta: { title: 'JumpServer 同步', icon: 'el-icon-copy-document' } + }, { path: 'exp', name: '余额与到期通知', diff --git a/vue-consul/src/views/jms/index.vue b/vue-consul/src/views/jms/index.vue new file mode 100644 index 0000000..efda7ca --- /dev/null +++ b/vue-consul/src/views/jms/index.vue @@ -0,0 +1,147 @@ + + +