From fd9f527cbc267cceb4cea3c4031c2e40e1e6c8e3 Mon Sep 17 00:00:00 2001 From: starsliao Date: Wed, 7 Dec 2022 01:25:09 +0800 Subject: [PATCH] fix bug add logger --- flask-consul/config.py | 1 + flask-consul/manager.py | 3 +- flask-consul/requirements.txt | 1 + flask-consul/units/avd/avd_list.py | 11 ++--- flask-consul/units/cloud/alicloud.py | 29 ++++++------ flask-consul/units/cloud/huaweicloud.py | 55 ++++++++++++----------- flask-consul/units/cloud/notify.py | 7 +-- flask-consul/units/cloud/sync_ecs.py | 9 ++-- flask-consul/units/cloud/sync_rds.py | 8 ++-- flask-consul/units/cloud/sync_redis.py | 8 ++-- flask-consul/units/cloud/tencent_cloud.py | 24 +++++----- flask-consul/units/config_log.py | 5 +++ flask-consul/units/consul_manager.py | 3 +- flask-consul/units/consul_svc.py | 4 +- flask-consul/units/jms/sync_jms.py | 23 +++++----- flask-consul/units/ldap/LdapUser.py | 15 +++---- flask-consul/units/prom/mysql_ali.py | 2 - flask-consul/units/prom/mysql_huawei.py | 51 +++++++++------------ flask-consul/units/prom/mysql_tencent.py | 40 ++++++++--------- flask-consul/units/prom/redis_ali.py | 2 - flask-consul/units/prom/redis_huawei.py | 51 +++++++++------------ flask-consul/units/prom/redis_tencent.py | 47 +++++++++---------- flask-consul/units/selfnode_manager.py | 3 +- flask-consul/units/selfrds_manager.py | 4 +- flask-consul/units/selfredis_manager.py | 5 +-- flask-consul/units/token_auth.py | 3 +- flask-consul/units/upload.py | 15 ++++--- flask-consul/views/blackbox.py | 4 +- flask-consul/views/edit_cloud.py | 3 +- flask-consul/views/jms.py | 3 +- flask-consul/views/login.py | 5 ++- flask-consul/views/nodes.py | 4 +- flask-consul/views/rds.py | 4 +- flask-consul/views/redis.py | 4 +- flask-consul/views/selfnode.py | 6 +-- flask-consul/views/selfrds.py | 6 +-- flask-consul/views/selfredis.py | 6 +-- 37 files changed, 226 insertions(+), 248 deletions(-) create mode 100644 flask-consul/units/config_log.py diff --git a/flask-consul/config.py b/flask-consul/config.py index 1ce124c..33af862 100644 --- a/flask-consul/config.py +++ b/flask-consul/config.py @@ -3,6 +3,7 @@ import os consul_token = os.environ.get('consul_token','0a79caed-8a45-49b9-97a6-86e50e12b234') consul_url = os.environ.get('consul_url','http://10.5.148.67:8500/v1') admin_passwd = os.environ.get('admin_passwd','123456') +log_level = os.environ.get('log_level','INFO') vendors = {'alicloud': '阿里云','tencent_cloud': '腾讯云','huaweicloud': '华为云'} regions = {'huaweicloud':{'none': '无','cn-east-3': '华东-上海一','cn-east-2': '华东-上海二', diff --git a/flask-consul/manager.py b/flask-consul/manager.py index 67440a7..80bce82 100755 --- a/flask-consul/manager.py +++ b/flask-consul/manager.py @@ -2,6 +2,7 @@ from flask import Flask from units import consul_kv import uuid +from units.config_log import * skey_path = 'ConsulManager/assets/secret/skey' if consul_kv.get_kv_dict(skey_path) == {}: @@ -46,7 +47,7 @@ init_jobs = { **ecs_jobs, **avd_jobs, **exp_jobs, **jms_jobs } if init_jobs is not None: for k,v in init_jobs.items(): - print(f"【初始化任务】{k}:{v['args']},{v['minutes']}m", flush=True) + logger.info(f"初始化任务:{k}:{v['args']},{v['minutes']}m") Config.JOBS = init_jobs.values() app.config.from_object(Config()) diff --git a/flask-consul/requirements.txt b/flask-consul/requirements.txt index d5bc1ea..56d6c9a 100644 --- a/flask-consul/requirements.txt +++ b/flask-consul/requirements.txt @@ -11,6 +11,7 @@ xlrd==1.2.0 pycryptodome==3.14.1 beautifulsoup4==4.11.1 ldap3==2.9.1 +loguru==0.6.0 huaweicloudsdkcore==3.1.11 huaweicloudsdkecs==3.1.11 huaweicloudsdkeps==3.1.11 diff --git a/flask-consul/units/avd/avd_list.py b/flask-consul/units/avd/avd_list.py index 7d3e8de..0137fda 100644 --- a/flask-consul/units/avd/avd_list.py +++ b/flask-consul/units/avd/avd_list.py @@ -2,6 +2,7 @@ import sys,requests,hashlib,json from datetime import datetime from bs4 import BeautifulSoup from units import consul_kv +from units.config_log import * def get_avd(): avd_url = 'https://avd.aliyun.com' @@ -23,13 +24,13 @@ def get_avd(): avd_dict['avd_time'] = avd[3].getText(strip=True) avd_dict['avd_stat'] = avd[4].select('button')[1].attrs['title'] if index == 0 and avd_dict == last_avd: - print('【JOB】===>','avd_list','未采集到新漏洞。',flush=True) + logger.info('【JOB】===> avd_list 未采集到新漏洞。') break else: avd_dict['avd_collect'] = now consul_kv.put_kv(f'ConsulManager/avd/list/{index}',avd_dict) if index == 0: - print('【JOB】===>','avd_list',avd_dict,flush=True) + logger.info(f'【JOB】===> avd_list {avd_dict}') avd_switch = consul_kv.get_value('ConsulManager/avd/switch') wecomwh = avd_switch.get('wecomwh','') dingdingwh = avd_switch.get('dingdingwh','') @@ -60,14 +61,14 @@ def wecom(webhook,content): params = {'msgtype': 'markdown', 'markdown': {'content' : content}} data = bytes(json.dumps(params), 'utf-8') response = requests.post(webhook, headers=headers, data=data) - print('【wecom】',response.json(),flush=True) + logger.info(f'【wecom】{response.json()}') def dingding(webhook,content): headers = {'Content-Type': 'application/json'} params = {"msgtype":"markdown","markdown":{"title":"漏洞告警","text":content},"at":{"isAtAll":True}} data = bytes(json.dumps(params), 'utf-8') response = requests.post(webhook, headers=headers, data=data) - print('【dingding】',response.json(),flush=True) + logger.info(f'【dingding】{response.json()}') def feishu(webhook,title,md): headers = {'Content-Type': 'application/json'} @@ -76,4 +77,4 @@ def feishu(webhook,title,md): "elements": [{"tag": "markdown","content": f"{md}\n",}]}} data = json.dumps(params) response = requests.post(webhook, headers=headers, data=data) - print('【feishu】',response.json(),flush=True) + logger.info(f'【feishu】{response.json()}') diff --git a/flask-consul/units/cloud/alicloud.py b/flask-consul/units/cloud/alicloud.py index 0dc4bee..eccd7e1 100644 --- a/flask-consul/units/cloud/alicloud.py +++ b/flask-consul/units/cloud/alicloud.py @@ -17,9 +17,10 @@ from alibabacloud_r_kvstore20150101.client import Client as R_kvstore20150101Cli import sys,datetime,hashlib from units import consul_kv,consul_svc from units.cloud import sync_ecs,sync_rds,sync_redis,notify +from units.config_log import * def exp(account,collect_days,notify_days,notify_amount): - #print(f"=====【阿里云:余额与到期日统计开始:{account}】", flush=True) + logger.debug(f"=====【阿里云:余额与到期日统计开始:{account}】") ak,sk = consul_kv.get_aksk('alicloud',account) now = datetime.datetime.utcnow().strftime('%Y-%m-%dT16:00:00Z') collect = (datetime.datetime.utcnow() + datetime.timedelta(days=collect_days+1)).strftime('%Y-%m-%dT16:00:00Z') @@ -37,7 +38,7 @@ def exp(account,collect_days,notify_days,notify_amount): available_amount = amount_response.body.data.available_amount amount = float(available_amount.replace(',','')) consul_kv.put_kv(f'ConsulManager/exp/lists/alicloud/{account}/amount',{'amount':amount}) - #print('alicloud',account,f'可用余额:{available_amount}', flush=True) + logger.debug(f'alicloud {account} 可用余额:{available_amount}') amount_dict = {} if amount < notify_amount: amount_dict = {'amount':amount} @@ -51,9 +52,9 @@ def exp(account,collect_days,notify_days,notify_amount): md = content notify.feishu(feishuwh,title,md,isatall) else: - print(f'查询失败,Code:{amount_response.body.code}, 信息:{amount_response.body.message}, requestId:{amount_response.body.request_id}', flush=True) + logger.error(f'查询失败,Code:{amount_response.body.code}, 信息:{amount_response.body.message}, requestId:{amount_response.body.request_id}') except Exception as e: - print('==ERROR==',e,flush=True) + logger.error(f'==ERROR=={e}') raise query_available_instances_request = bss_open_api_20171214_models.QueryAvailableInstancesRequest(renew_status='ManualRenewal',end_time_start=now,end_time_end=collect) runtime = util_models.RuntimeOptions() @@ -62,7 +63,7 @@ def exp(account,collect_days,notify_days,notify_amount): exp_list = exp.body.to_map()['Data']['InstanceList'] except Exception as e: #exp_list = [] - print('==ERROR==',e,flush=True) + logger.error(f'==ERROR=={e}') raise exp_dict = {} isnotify_list = consul_kv.get_keys_list(f'ConsulManager/exp/isnotify/alicloud/{account}') @@ -93,7 +94,7 @@ def exp(account,collect_days,notify_days,notify_amount): title = '阿里云资源到期通知' md = content notify.feishu(feishuwh,title,md,isatall) - #print(f"=====【阿里云:余额与到期日统计结束:{account}】", flush=True) + logger.debug(f"=====【阿里云:余额与到期日统计结束:{account}】") def group(account): ak,sk = consul_kv.get_aksk('alicloud',account) @@ -110,10 +111,10 @@ def group(account): count = len(group_dict) data = {'count':count,'update':now,'status':20000,'msg':f'同步资源组成功!总数:{count}'} consul_kv.put_kv(f'ConsulManager/record/jobs/alicloud/{account}/group', data) - print('【JOB】===>', 'alicloud_group', account, data, flush=True) + logger.info(f'【JOB】===>alicloud_group {account} {data}') except TeaException as e: emsg = e.message.split('. ',1)[0] - print("【code:】",e.code,"\n【message:】",emsg, flush=True) + logger.error(f"【code:】{e.code}\n【message:】{emsg}") data = consul_kv.get_value(f'ConsulManager/record/jobs/alicloud/{account}/group') if data == {}: data = {'count':'无','update':f'失败{e.code}','status':50000,'msg':emsg} @@ -164,10 +165,10 @@ def ecs(account,region,isextip=False): off,on = sync_ecs.w2consul('alicloud',account,region,ecs_dict) data = {'count':count,'update':now,'status':20000,'on':on,'off':off,'msg':f'ECS同步成功!总数:{count},开机:{on},关机:{off}'} consul_kv.put_kv(f'ConsulManager/record/jobs/alicloud/{account}/ecs/{region}', data) - print('【JOB】===>', 'alicloud_ecs', account,region, data, flush=True) + logger.info(f'【JOB】===>alicloud_ecs {account} {region} {data}') except TeaException as e: emsg = e.message.split('. ',1)[0] - print("【code:】",e.code,"\n【message:】",emsg, flush=True) + logger.error(f"【code:】{e.code}\n【message:】{emsg}") data = consul_kv.get_value(f'ConsulManager/record/jobs/alicloud/{account}/ecs/{region}') if data == {}: data = {'count':'无','update':f'失败{e.code}','status':50000,'msg':emsg} @@ -214,10 +215,10 @@ def redis(account,region): off,on = sync_redis.w2consul('alicloud',account,region,redis_dict) data = {'count':count,'update':now,'status':20000,'on':on,'off':off,'msg':f'redis同步成功!总数:{count},开机:{on},关机:{off}'} consul_kv.put_kv(f'ConsulManager/record/jobs/alicloud/{account}/redis/{region}', data) - print('【JOB】===>', 'alicloud_redis', account,region, data, flush=True) + logger.info(f'【JOB】===>alicloud_redis {account} {region} {data}') except TeaException as e: emsg = e.message.split('. ',1)[0] - print("【code:】",e.code,"\n【message:】",emsg, flush=True) + logger.error(f"【code:】{e.code}\n【message:】{emsg}") data = consul_kv.get_value(f'ConsulManager/record/jobs/alicloud/{account}/redis/{region}') if data == {}: data = {'count':'无','update':f'失败{e.code}','status':50000,'msg':emsg} @@ -276,10 +277,10 @@ def rds(account,region): off,on = sync_rds.w2consul('alicloud',account,region,rds_dict) data = {'count':count,'update':now,'status':20000,'on':on,'off':off,'msg':f'rds同步成功!总数:{count},开机:{on},关机:{off}'} consul_kv.put_kv(f'ConsulManager/record/jobs/alicloud/{account}/rds/{region}', data) - print('【JOB】===>', 'alicloud_rds', account,region, data, flush=True) + logger.info(f'【JOB】===>alicloud_rds {account} {region} {data}') except TeaException as e: emsg = e.message.split('. ',1)[0] - print("【code:】",e.code,"\n【message:】",emsg, flush=True) + logger.error(f"【code:】{e.code}\n【message:】{emsg}") data = consul_kv.get_value(f'ConsulManager/record/jobs/alicloud/{account}/rds/{region}') if data == {}: data = {'count':'无','update':f'失败{e.code}','status':50000,'msg':emsg} diff --git a/flask-consul/units/cloud/huaweicloud.py b/flask-consul/units/cloud/huaweicloud.py index 94af995..70a4199 100644 --- a/flask-consul/units/cloud/huaweicloud.py +++ b/flask-consul/units/cloud/huaweicloud.py @@ -6,16 +6,17 @@ from huaweicloudsdkeps.v1 import * from huaweicloudsdkbss.v2 import * from huaweicloudsdkecs.v2 import * from huaweicloudsdkecs.v2.region.ecs_region import EcsRegion -from huaweicloudsdkrds.v3 import * -from huaweicloudsdkrds.v3.region.rds_region import RdsRegion from huaweicloudsdkdcs.v2 import * from huaweicloudsdkdcs.v2.region.dcs_region import DcsRegion +from huaweicloudsdkrds.v3 import * +from huaweicloudsdkrds.v3.region.rds_region import RdsRegion import sys,datetime,hashlib from units import consul_kv from units.cloud import sync_ecs from units.cloud import sync_rds from units.cloud import sync_redis from units.cloud import notify +from units.config_log import * def exp(account,collect_days,notify_days,notify_amount): ak,sk = consul_kv.get_aksk('huaweicloud',account) @@ -90,10 +91,10 @@ def exp(account,collect_days,notify_days,notify_amount): notify.feishu(feishuwh,title,md,isatall) except exceptions.ClientRequestException as e: - print(e.status_code) - print(e.request_id) - print(e.error_code) - print(e.error_msg) + logger.error(e.status_code) + logger.error(e.request_id) + logger.error(e.error_code) + logger.error(e.error_msg) def group(account): ak,sk = consul_kv.get_aksk('huaweicloud',account) @@ -113,12 +114,12 @@ def group(account): count = len(group_dict) data = {'count':count,'update':now,'status':20000,'msg':f'同步企业项目成功!总数:{count}'} consul_kv.put_kv(f'ConsulManager/record/jobs/huaweicloud/{account}/group', data) - print('【JOB】===>', 'huaweicloud_group', account, data, flush=True) + logger.info(f'【JOB】===>huaweicloud_group {account} {data}') except exceptions.ClientRequestException as e: - print(e.status_code, flush=True) - print(e.request_id, flush=True) - print(e.error_code, flush=True) - print(e.error_msg, flush=True) + logger.error(e.status_code) + logger.error(e.request_id) + logger.error(e.error_code) + logger.error(e.error_msg) data = consul_kv.get_value(f'ConsulManager/record/jobs/huaweicloud/{account}/group') if data == {}: data = {'count':'无','update':f'失败{e.status_code}','status':50000,'msg':e.error_msg} @@ -157,12 +158,12 @@ def ecs(account,region,isextip=False): off,on = sync_ecs.w2consul('huaweicloud',account,region,ecs_dict) data = {'count':count,'update':now,'status':20000,'on':on,'off':off,'msg':f'ECS同步成功!总数:{count},开机:{on},关机:{off}'} consul_kv.put_kv(f'ConsulManager/record/jobs/huaweicloud/{account}/ecs/{region}', data) - print('【JOB】===>', 'huaweicloud_ecs', account,region, data, flush=True) + logger.info(f'【JOB】===>huaweicloud_ecs {account} {region} {data}') except exceptions.ClientRequestException as e: - print(e.status_code, flush=True) - print(e.request_id, flush=True) - print(e.error_code, flush=True) - print(e.error_msg, flush=True) + logger.error(e.status_code) + logger.error(e.request_id) + logger.error(e.error_code) + logger.error(e.error_msg) data = consul_kv.get_value(f'ConsulManager/record/jobs/huaweicloud/{account}/ecs/{region}') if data == {}: data = {'count':'无','update':f'失败{e.status_code}','status':50000,'on':0,'off':0,'msg':e.error_msg} @@ -203,17 +204,17 @@ def rds(account,region): 'mem':f"{i['mem']}GB", 'disk':f"{i['volume']['size']}GB", 'exp': '-' if i['expiration_time'] == None else i['expiration_time'].split('T')[0] - } for i in info} + } for i in info if i['datastore']['type'] == 'MySQL'} count = len(rds_dict) off,on = sync_rds.w2consul('huaweicloud',account,region,rds_dict) data = {'count':count,'update':now,'status':20000,'on':on,'off':off,'msg':f'RDS同步成功!总数:{count},开机:{on},关机:{off}'} consul_kv.put_kv(f'ConsulManager/record/jobs/huaweicloud/{account}/rds/{region}', data) - print('【JOB】===>', 'huaweicloud_rds', account,region, data, flush=True) + logger.info(f'【JOB】===>huaweicloud_rds {account} {region} {data}') except exceptions.ClientRequestException as e: - print(e.status_code, flush=True) - print(e.request_id, flush=True) - print(e.error_code, flush=True) - print(e.error_msg, flush=True) + logger.error(e.status_code) + logger.error(e.request_id) + logger.error(e.error_code) + logger.error(e.error_msg) data = consul_kv.get_value(f'ConsulManager/record/jobs/huaweicloud/{account}/rds/{region}') if data == {}: data = {'count':'无','update':f'失败{e.status_code}','status':50000,'on':0,'off':0,'msg':e.error_msg} @@ -257,12 +258,12 @@ def redis(account,region): off,on = sync_redis.w2consul('huaweicloud',account,region,redis_dict) data = {'count':count,'update':now,'status':20000,'on':on,'off':off,'msg':f'REDIS同步成功!总数:{count},开机:{on},关机:{off}'} consul_kv.put_kv(f'ConsulManager/record/jobs/huaweicloud/{account}/redis/{region}', data) - print('【JOB】===>', 'huaweicloud_redis', account,region, data, flush=True) + logger.info(f'【JOB】===>huaweicloud_redis {account} {region} {data}') except exceptions.ClientRequestException as e: - print(e.status_code, flush=True) - print(e.request_id, flush=True) - print(e.error_code, flush=True) - print(e.error_msg, flush=True) + logger.error(e.status_code) + logger.error(e.request_id) + logger.error(e.error_code) + logger.error(e.error_msg) data = consul_kv.get_value(f'ConsulManager/record/jobs/huaweicloud/{account}/redis/{region}') if data == {}: data = {'count':'无','update':f'失败{e.status_code}','status':50000,'on':0,'off':0,'msg':e.error_msg} diff --git a/flask-consul/units/cloud/notify.py b/flask-consul/units/cloud/notify.py index 8e81e30..5d6c3e5 100644 --- a/flask-consul/units/cloud/notify.py +++ b/flask-consul/units/cloud/notify.py @@ -1,17 +1,18 @@ import requests,json +from units.config_log import * def wecom(webhook,content): headers = {'Content-Type': 'application/json'} params = {'msgtype': 'markdown', 'markdown': {'content' : content}} data = bytes(json.dumps(params), 'utf-8') response = requests.post(webhook, headers=headers, data=data) - print('【wecom】',response.json(),flush=True) + logger.info(f'【wecom】{response.json()}') def dingding(webhook,content,isatall=True): headers = {'Content-Type': 'application/json'} params = {"msgtype":"markdown","markdown":{"title":"资源告警","text":content},"at":{"isAtAll":isatall}} data = bytes(json.dumps(params), 'utf-8') response = requests.post(webhook, headers=headers, data=data) - print('【dingding】',response.json(),flush=True) + logger.info(f'【dingding】{response.json()}') def feishu(webhook,title,md,isatall=True): headers = {'Content-Type': 'application/json'} @@ -21,4 +22,4 @@ def feishu(webhook,title,md,isatall=True): "elements": [{"tag": "markdown","content": f"{md}\n{atall}",}]}} data = json.dumps(params) response = requests.post(webhook, headers=headers, data=data) - print('【feishu】',response.json(),flush=True) + logger.info(f'【feishu】{response.json()}') diff --git a/flask-consul/units/cloud/sync_ecs.py b/flask-consul/units/cloud/sync_ecs.py index fa154a5..bd06eac 100644 --- a/flask-consul/units/cloud/sync_ecs.py +++ b/flask-consul/units/cloud/sync_ecs.py @@ -2,6 +2,7 @@ import requests,json from units import consul_kv from config import consul_token,consul_url,vendors,regions +from units.config_log import * headers = {'X-Consul-Token': consul_token} geturl = f'{consul_url}/agent/services' delurl = f'{consul_url}/agent/service/deregister' @@ -18,9 +19,9 @@ def w2consul(vendor,account,region,ecs_dict): for del_ecs in [x for x in consul_ecs_iid_list if x not in ecs_dict.keys()]: dereg = requests.put(f'{delurl}/{del_ecs}', headers=headers) if dereg.status_code == 200: - print({"code": 20000,"data": f"{account}-删除成功!"}, flush=True) + logger.info(f"code: 20000, data: {account}-删除成功!") else: - print({"code": 50000,"data": f'{dereg.status_code}:{dereg.text}'}, flush=True) + logger.info(f"code: 50000, data: {dereg.status_code}:{dereg.text}") off,on = 0,0 for k,v in ecs_dict.items(): iid = k @@ -75,8 +76,6 @@ def w2consul(vendor,account,region,ecs_dict): reg = requests.put(puturl, headers=headers, data=json.dumps(data)) if reg.status_code == 200: pass - #print({f"{account}:code": 20000,"data": "增加成功!"}, flush=True) else: - print({f"{account}:code": 50000,"data": f'{reg.status_code}:{reg.text}'}, flush=True) - #return {"code": 50000,"data": f'{reg.status_code}:{reg.text}'} + logger.info(f"{account}:code: 5000, data: {reg.status_code}:{reg.text}") return off,on diff --git a/flask-consul/units/cloud/sync_rds.py b/flask-consul/units/cloud/sync_rds.py index e6edbe4..1298108 100644 --- a/flask-consul/units/cloud/sync_rds.py +++ b/flask-consul/units/cloud/sync_rds.py @@ -18,9 +18,9 @@ def w2consul(vendor,account,region,rds_dict): for del_rds in [x for x in consul_rds_iid_list if x not in rds_dict.keys()]: dereg = requests.put(f'{delurl}/{del_rds}', headers=headers) if dereg.status_code == 200: - print({"code": 20000,"data": f"{account}-删除成功!"}, flush=True) + logger.info(f"code: 20000, data: {account}-删除成功!") else: - print({"code": 50000,"data": f'{dereg.status_code}:{dereg.text}'}, flush=True) + logger.info(f"code: 50000, data: {dereg.status_code}:{dereg.text}") off,on = 0,0 for k,v in rds_dict.items(): iid = k @@ -72,8 +72,6 @@ def w2consul(vendor,account,region,rds_dict): reg = requests.put(puturl, headers=headers, data=json.dumps(data)) if reg.status_code == 200: pass - #print({f"{account}:code": 20000,"data": "增加成功!"}, flush=True) else: - print({f"{account}:code": 50000,"data": f'{reg.status_code}:{reg.text}'}, flush=True) - #return {"code": 50000,"data": f'{reg.status_code}:{reg.text}'} + logger.info(f"{account}:code: 5000, data: {reg.status_code}:{reg.text}") return off,on diff --git a/flask-consul/units/cloud/sync_redis.py b/flask-consul/units/cloud/sync_redis.py index 3821403..2c05a9c 100644 --- a/flask-consul/units/cloud/sync_redis.py +++ b/flask-consul/units/cloud/sync_redis.py @@ -18,9 +18,9 @@ def w2consul(vendor,account,region,redis_dict): for del_redis in [x for x in consul_redis_iid_list if x not in redis_dict.keys()]: dereg = requests.put(f'{delurl}/{del_redis}', headers=headers) if dereg.status_code == 200: - print({"code": 20000,"data": f"{account}-删除成功!"}, flush=True) + logger.info(f"code: 20000, data: {account}-删除成功!") else: - print({"code": 50000,"data": f'{dereg.status_code}:{dereg.text}'}, flush=True) + logger.info(f"code: 50000, data: {dereg.status_code}:{dereg.text}") off,on = 0,0 for k,v in redis_dict.items(): iid = k @@ -70,8 +70,6 @@ def w2consul(vendor,account,region,redis_dict): reg = requests.put(puturl, headers=headers, data=json.dumps(data)) if reg.status_code == 200: pass - #print({f"{account}:code": 20000,"data": "增加成功!"}, flush=True) else: - print({f"{account}:code": 50000,"data": f'{reg.status_code}:{reg.text}'}, flush=True) - #return {"code": 50000,"data": f'{reg.status_code}:{reg.text}'} + logger.info(f"{account}:code: 5000, data: {reg.status_code}:{reg.text}") return off,on diff --git a/flask-consul/units/cloud/tencent_cloud.py b/flask-consul/units/cloud/tencent_cloud.py index be8941e..46be064 100644 --- a/flask-consul/units/cloud/tencent_cloud.py +++ b/flask-consul/units/cloud/tencent_cloud.py @@ -8,10 +8,8 @@ import sys,datetime,hashlib #sys.path.append("..") #import consul_kv,sync_ecs from units import consul_kv -from units.cloud import sync_ecs -from units.cloud import sync_rds -from units.cloud import sync_redis -from units.cloud import notify +from units.cloud import sync_ecs,sync_rds,sync_redis,notify +from units.config_log import * def exp(account,collect_days,notify_days,notify_amount): from tencentcloud.billing.v20180709 import billing_client, models @@ -79,7 +77,7 @@ def exp(account,collect_days,notify_days,notify_amount): md = content notify.feishu(feishuwh,title,md,isatall) except TencentCloudSDKException as err: - print(err) + logger.error(f'{err}') def group(account): from tencentcloud.dcdb.v20180411 import dcdb_client, models @@ -102,9 +100,9 @@ def group(account): count = len(group_dict) data = {'count':count,'update':now,'status':20000,'msg':f'同步资源组成功!总数:{count}'} consul_kv.put_kv(f'ConsulManager/record/jobs/tencent_cloud/{account}/group', data) - print('【JOB】===>', 'tencent_cloud_group', account, data, flush=True) + logger.info(f'【JOB】===>tencent_cloud_group {account} {data}') except TencentCloudSDKException as err: - print(err, flush=True) + logger.error(f'{err}') data = consul_kv.get_value(f'ConsulManager/record/jobs/tencent_cloud/{account}/group') if data == {}: data = {'count':'无','update':f'失败','status':50000,'msg':str(err)} @@ -152,9 +150,9 @@ def ecs(account,region,isextip=False): off,on = sync_ecs.w2consul('tencent_cloud',account,region,ecs_dict) data = {'count':count,'update':now,'status':20000,'on':on,'off':off,'msg':f'ECS同步成功!总数:{count},开机:{on},关机:{off}'} consul_kv.put_kv(f'ConsulManager/record/jobs/tencent_cloud/{account}/ecs/{region}', data) - print('【JOB】===>', 'tencent_cloud_ecs', account,region, data, flush=True) + logger.info(f'【JOB】===>tencent_cloud_ecs {account} {region} {data}') except TencentCloudSDKException as err: - print(err, flush=True) + logger.error(f'{err}') data = consul_kv.get_value(f'ConsulManager/record/jobs/tencent_cloud/{account}/ecs/{region}') if data == {}: data = {'count':'无','update':f'失败','status':50000,'msg':str(err)} @@ -203,9 +201,9 @@ def rds(account,region): off,on = sync_rds.w2consul('tencent_cloud',account,region,rds_dict) data = {'count':count,'update':now,'status':20000,'on':on,'off':off,'msg':f'rds同步成功!总数:{count},开机:{on},关机:{off}'} consul_kv.put_kv(f'ConsulManager/record/jobs/tencent_cloud/{account}/rds/{region}', data) - print('【JOB】===>', 'tencent_cloud_rds', account,region, data, flush=True) + logger.info(f'【JOB】===>tencent_cloud_rds {account} {region} {data}') except TencentCloudSDKException as err: - print(err, flush=True) + logger.error(f'{err}') data = consul_kv.get_value(f'ConsulManager/record/jobs/tencent_cloud/{account}/rds/{region}') if data == {}: data = {'count':'无','update':f'失败','status':50000,'msg':str(err)} @@ -252,9 +250,9 @@ def redis(account,region): off,on = sync_redis.w2consul('tencent_cloud',account,region,redis_dict) data = {'count':count,'update':now,'status':20000,'on':on,'off':off,'msg':f'redis同步成功!总数:{count},开机:{on},关机:{off}'} consul_kv.put_kv(f'ConsulManager/record/jobs/tencent_cloud/{account}/redis/{region}', data) - print('【JOB】===>', 'tencent_cloud_redis', account,region, data, flush=True) + logger.info(f'【JOB】===>tencent_cloud_redis {account} {region} {data}') except TencentCloudSDKException as err: - print(err, flush=True) + logger.error(f'{err}') data = consul_kv.get_value(f'ConsulManager/record/jobs/tencent_cloud/{account}/redis/{region}') if data == {}: data = {'count':'无','update':f'失败','status':50000,'msg':str(err)} diff --git a/flask-consul/units/config_log.py b/flask-consul/units/config_log.py new file mode 100644 index 0000000..c7118fa --- /dev/null +++ b/flask-consul/units/config_log.py @@ -0,0 +1,5 @@ +from config import log_level +import sys +from loguru import logger +logger.remove() +logger.add(sys.stderr,format='{time:HH:mm:ss} | {level} | {message}',level=log_level) diff --git a/flask-consul/units/consul_manager.py b/flask-consul/units/consul_manager.py index 91c7e12..d3cba45 100644 --- a/flask-consul/units/consul_manager.py +++ b/flask-consul/units/consul_manager.py @@ -2,6 +2,7 @@ import requests,json import sys sys.path.append("..") from config import consul_token,consul_url +from units.config_log import * headers = {'X-Consul-Token': consul_token} @@ -112,7 +113,7 @@ def add_instance(instance_dict): del instance_dict['metaInfo'] del instance_dict['checkInfo'] - print(instance_dict, flush=True) + logger.info(f'{instance_dict}') reg = requests.put(f'{consul_url}/agent/service/register', headers=headers, data=json.dumps(instance_dict)) if reg.status_code == 200: diff --git a/flask-consul/units/consul_svc.py b/flask-consul/units/consul_svc.py index 1ef786b..b6c4517 100644 --- a/flask-consul/units/consul_svc.py +++ b/flask-consul/units/consul_svc.py @@ -2,7 +2,7 @@ import requests,json import sys sys.path.append("..") from config import consul_token,consul_url - +from units.config_log import * headers = {'X-Consul-Token': consul_token} def get_sid(iid): @@ -26,6 +26,6 @@ def add_sid(instance_dict): if reg.status_code == 200: return {"code": 20000, "data": f"增加成功!"} else: - print(f"{reg.status_code}:{reg.text}") + logger.info(f"{reg.status_code}:{reg.text}") return {"code": 50000, "data": f"{reg.status_code}:{reg.text}"} diff --git a/flask-consul/units/jms/sync_jms.py b/flask-consul/units/jms/sync_jms.py index 4b23c35..576518f 100755 --- a/flask-consul/units/jms/sync_jms.py +++ b/flask-consul/units/jms/sync_jms.py @@ -1,5 +1,6 @@ import datetime,requests,json from units import consul_kv,consul_manager,myaes +from units.config_log import * #创建node def create_node(jms_url,headers,now,node_id,cloud,account): @@ -7,7 +8,7 @@ def create_node(jms_url,headers,now,node_id,cloud,account): jms_node_list = requests.request("GET", node_url, headers=headers).json() if type(jms_node_list) == dict: detail = jms_node_list.get('detail','ERROR') - print(' 【JMS】',detail,flush=True) + logger.info(f' 【JMS】{detail}') data = {'count': '失败','update':now,'status':50000,'msg':f'同步资源失败!{detail}'} consul_kv.put_kv(f'ConsulManager/record/jms/{cloud}/{account}', data) cloud_group_dict = consul_kv.get_value(f'ConsulManager/assets/{cloud}/group/{account}') @@ -15,7 +16,7 @@ def create_node(jms_url,headers,now,node_id,cloud,account): for k,v in cloud_group_dict.items(): if v not in [i['value'] for i in jms_node_list]: response = requests.request("POST", node_url, headers=headers, data = json.dumps({'value': v})) - print(' 【JMS】新增组===>',v,response.status_code,flush=True) + logger.info(f' 【JMS】新增组===>{v}{response.status_code}') reget_node_list = requests.request("GET", node_url, headers=headers).json() new_node_dict = {i['value']:i['id'] for i in reget_node_list} return new_node_dict @@ -31,7 +32,7 @@ def update_jms_ecs(jms_url,headers,new_node_dict,node_id,cloud,account,ecs_info, del_ecs_list = [v['id'] for k,v in jms_ecs_dict.items() if k not in [i['ip'] for i in ecs_dict.values()]] for del_ecs in del_ecs_list: response = requests.request("DELETE", f'{ecs_url}{del_ecs}/', headers=headers) - print(' 【JMS】删除主机:',del_ecs,response.status_code,flush=True) + logger.info(f' 【JMS】删除主机:{del_ecs}{response.status_code}') #增加/更新缺少的主机 for k,v in ecs_dict.items(): @@ -66,13 +67,13 @@ def update_jms_ecs(jms_url,headers,new_node_dict,node_id,cloud,account,ecs_info, jms_group = '无' if jms_ecs_dict[ip]['node'].split('/')[-1] == '未分组' else jms_ecs_dict[ip]['node'].split('/')[-1] if jms_ecs_dict[ip]['name'] != iname or jms_group != v['ent']: response = requests.request("PUT", f"{ecs_url}{jms_ecs_dict[ip]['id']}/", headers=headers, data = json.dumps(payload)) - print(' 【JMS】update:主机名:',response.json()['hostname'],response.status_code,flush=True) + logger.info(f" 【JMS】update:主机名:{response.json()['hostname']}{response.status_code}") else: response = requests.request("POST", ecs_url, headers=headers, data = json.dumps(payload)) - print(' 【JMS】add:主机名:',iname,ip,f"【{response.json()['hostname']},{response.status_code}】",flush=True) + logger.info(f" 【JMS】add:主机名:{iname} {ip}【{response.json()['hostname']},{response.status_code}】") except Exception as e: - print('【update_jms ERROR】',e,flush=True) - print(response.json(),flush=True) + logger.error(f'【update_jms ERROR】{e}') + logger.error(f'{response.json()}') return ecs_ip_dict #从JMS中删除IP重复的主机 @@ -92,7 +93,7 @@ def del_jms_repip(jms_url,headers,node_id,ecs_ip_dict): if j['name'] != ecs_ip_dict.get(j['ip']): del_ecs = j['id'] response = requests.request("DELETE", f'{ecs_url}{del_ecs}/', headers=headers) - print(' 【JMS】删除IP重复且名称不在ECS列表的主机:',j['name'],j['ip'],response.status_code,flush=True) + logger.info(f" 【JMS】删除IP重复且名称不在ECS列表的主机:{j['name']},{j['ip']},{response.status_code}") #从JMS中删除没有主机的组 def del_node(jms_url,headers,now,node_id,cloud,account): @@ -105,7 +106,7 @@ def del_node(jms_url,headers,now,node_id,cloud,account): else: del_node_url = f"{jms_url}/api/v1/assets/nodes/{i['meta']['data']['id']}/" response = requests.request("DELETE", del_node_url, headers=headers) - print(' 【JMS】删除空组===>',i['name'],response.status_code,flush=True) + logger.info(" 【JMS】删除空组===>{i['name']},{response.status_code}") ecs_count_url = f"{jms_url}/api/v1/assets/assets/?node={node_id}&limit=1&offset=1" ecs_count = requests.request("GET", ecs_count_url, headers=headers).json()['count'] data = {'count':ecs_count,'update':now,'status':20000,'msg':f'同步资源成功!总数:{ecs_count}'} @@ -114,7 +115,7 @@ def del_node(jms_url,headers,now,node_id,cloud,account): def run(cloud,account): now = datetime.datetime.now().strftime('%m%d/%H:%M') - print('【JOB】===>',cloud,account,'JMS同步开始',flush=True) + logger.info(f'【JOB】===>{cloud},{account},JMS同步开始') node_id = consul_kv.get_value(f'ConsulManager/jms/{cloud}/{account}/node_id')['node_id'] temp_ecs_info = consul_kv.get_value(f'ConsulManager/jms/{cloud}/{account}/ecs_info') ecs_info = consul_kv.get_value(f'ConsulManager/jms/ecs_info') if temp_ecs_info == {} else temp_ecs_info @@ -130,4 +131,4 @@ def run(cloud,account): ecs_ip_dict = update_jms_ecs(jms_url,headers,new_node_dict,node_id,cloud,account,ecs_info,custom_ecs_info) del_jms_repip(jms_url,headers,node_id,ecs_ip_dict) del_node(jms_url,headers,now,node_id,cloud,account) - print('【JOB】===>',cloud,account,'JMS同步完成',flush=True) + logger.info(f'【JOB】===>{cloud},{account},JMS同步完成') diff --git a/flask-consul/units/ldap/LdapUser.py b/flask-consul/units/ldap/LdapUser.py index b56492b..eb3f7f3 100644 --- a/flask-consul/units/ldap/LdapUser.py +++ b/flask-consul/units/ldap/LdapUser.py @@ -1,11 +1,6 @@ -""" -ldap 用户认证 -""" - from ldap3 import Server, Connection, ALL - from units.ldap.ldap_consul import Ldap_Consul - +from units.config_log import * class Ldap(object): def __init__(self,**args): @@ -22,7 +17,7 @@ class Ldap(object): return 0 if self.allow == '*' or username.lower() in self.allow.lower().split(','): ldap_username = self.ldapusr.format(username=username) - print('ldapuser:',ldap_username,flush=True) + logger.info(f'ldapuser: {ldap_username}') server = Server(self.ldap_url,port=self.port, get_info=ALL,connect_timeout=5) conn = Connection(server, user=ldap_username, password=password, check_names=True, lazy=False, raise_exceptions=False) try: @@ -78,13 +73,13 @@ class Ldap(object): {'cn': "user1", 'sn': 'user1',"employeeType":"developer", 'gidNumber': 501, 'homeDirectory': '/home/users/{0}', 'uidNumber': 5000,"givenName":"user1", "loginShell":"/bin/bash",'displayName': "测试用户",'userPassword': "111111", 'mail': 'user1@qq.com'}), - print(c) + logger.info(c) #删除用户 def delete_user(self): c = self.conn.delete('cn=xxx,ou=People,dc=xxx,dc=com') - print(c) + logger.info(c) # def __del__(self): # self.conn.delete() @@ -93,4 +88,4 @@ class Ldap(object): if __name__ == '__main__': ldap = Ldap() result = ldap.delete_user() - print(result) + logger.info(result) diff --git a/flask-consul/units/prom/mysql_ali.py b/flask-consul/units/prom/mysql_ali.py index 6ee2dbb..542ce5f 100644 --- a/flask-consul/units/prom/mysql_ali.py +++ b/flask-consul/units/prom/mysql_ali.py @@ -1,6 +1,4 @@ from aliyunsdkcore.client import AcsClient -from aliyunsdkcore.acs_exception.exceptions import ClientException -from aliyunsdkcore.acs_exception.exceptions import ServerException from aliyunsdkcms.request.v20190101.DescribeMetricLastRequest import DescribeMetricLastRequest from datetime import datetime from units import consul_kv diff --git a/flask-consul/units/prom/mysql_huawei.py b/flask-consul/units/prom/mysql_huawei.py index 93d25d0..5935c34 100644 --- a/flask-consul/units/prom/mysql_huawei.py +++ b/flask-consul/units/prom/mysql_huawei.py @@ -1,10 +1,8 @@ from huaweicloudsdkcore.auth.credentials import BasicCredentials from huaweicloudsdkces.v1.region.ces_region import CesRegion -from huaweicloudsdkcore.exceptions import exceptions from huaweicloudsdkces.v1 import * from datetime import datetime from units import consul_kv - def exporter(vendor,account,region): ak,sk = consul_kv.get_aksk(vendor,account) credentials = BasicCredentials(ak, sk) @@ -22,32 +20,25 @@ def exporter(vendor,account,region): metric_body_list = [] now = int(datetime.now().timestamp()*1000) rds_list = consul_kv.get_services_list_by_region(f'{vendor}_{account}_rds',region) - try: - for i in metric_name_dict.keys(): - for rdsid in rds_list: - metric_body_list.append(MetricInfo(namespace="SYS.RDS",metric_name=i,dimensions=[MetricsDimension(name="rds_cluster_id",value=rdsid)])) - - request = BatchListMetricDataRequest() - request.body = BatchListMetricDataRequestBody(to=now,_from=now-180000,filter="max",period="1",metrics=metric_body_list) - response = client.batch_list_metric_data(request).to_dict() - for i in response['metrics']: - rdsid= i['dimensions'][0]['value'] - try: - value = i['datapoints'][-1]['max'] - ts = i['datapoints'][-1]['timestamp'] - except: - value = -1 - ts = now - metric = i['metric_name'] - prom_metric_name = metric_name_dict[metric][0].split()[2] - metric_name_dict[metric].append(f'{prom_metric_name}{{iid="{rdsid}"}} {float(value)} {ts}') - prom_metric_list = [] - for x in metric_name_dict.values(): - prom_metric_list = prom_metric_list + x - return prom_metric_list - except exceptions.ClientRequestException as e: - print(e.status_code,flush=True) - print(e.request_id,flush=True) - print(e.error_code,flush=True) - print(e.error_msg,flush=True) + for i in metric_name_dict.keys(): + for rdsid in rds_list: + metric_body_list.append(MetricInfo(namespace="SYS.RDS",metric_name=i,dimensions=[MetricsDimension(name="rds_cluster_id",value=rdsid)])) + request = BatchListMetricDataRequest() + request.body = BatchListMetricDataRequestBody(to=now,_from=now-180000,filter="max",period="1",metrics=metric_body_list) + response = client.batch_list_metric_data(request).to_dict() + for i in response['metrics']: + rdsid= i['dimensions'][0]['value'] + try: + value = i['datapoints'][-1]['max'] + ts = i['datapoints'][-1]['timestamp'] + except: + value = -1 + ts = now + metric = i['metric_name'] + prom_metric_name = metric_name_dict[metric][0].split()[2] + metric_name_dict[metric].append(f'{prom_metric_name}{{iid="{rdsid}"}} {float(value)} {ts}') + prom_metric_list = [] + for x in metric_name_dict.values(): + prom_metric_list = prom_metric_list + x + return prom_metric_list diff --git a/flask-consul/units/prom/mysql_tencent.py b/flask-consul/units/prom/mysql_tencent.py index 8816ba2..85264f1 100644 --- a/flask-consul/units/prom/mysql_tencent.py +++ b/flask-consul/units/prom/mysql_tencent.py @@ -2,7 +2,6 @@ import json from tencentcloud.common import credential from tencentcloud.common.profile.client_profile import ClientProfile from tencentcloud.common.profile.http_profile import HttpProfile -from tencentcloud.common.exception.tencent_cloud_sdk_exception import TencentCloudSDKException from tencentcloud.monitor.v20180724 import monitor_client, models from datetime import datetime,timedelta from units import consul_kv @@ -21,24 +20,21 @@ def exporter(vendor,account,region): rds_list = consul_kv.get_services_list_by_region(f'{vendor}_{account}_rds',region) rds_list = list(rds_list) rds_list_10 = [rds_list[i:i + 10] for i in range(0, len(rds_list), 10)] - try: - for i in metric_name_dict.keys(): - for rdss in rds_list_10: - starttime = (datetime.now() + timedelta(minutes=-1)).strftime('%Y-%m-%dT%H:%M:%S+08:00') - ins_list = [{"Dimensions":[{"Name":"InstanceId","Value":x}]} for x in rdss] - params = {"Namespace":"QCE/CDB","MetricName":i,"Period":60,"StartTime":starttime,"Instances":ins_list} - req.from_json_string(json.dumps(params)) - resp = client.GetMonitorData(req) - metric_list = resp.DataPoints - for metrics in metric_list: - iid = metrics.Dimensions[0].Value - value = metrics.Values[-1] - ts = metrics.Timestamps[-1]*1000 - prom_metric_name = metric_name_dict[i][0].split()[2] - metric_name_dict[i].append(f'{prom_metric_name}{{iid="{iid}"}} {float(value)} {ts}') - prom_metric_list = [] - for x in metric_name_dict.values(): - prom_metric_list = prom_metric_list + x - return prom_metric_list - except TencentCloudSDKException as err: - print(err) + for i in metric_name_dict.keys(): + for rdss in rds_list_10: + starttime = (datetime.now() + timedelta(minutes=-1)).strftime('%Y-%m-%dT%H:%M:%S+08:00') + ins_list = [{"Dimensions":[{"Name":"InstanceId","Value":x}]} for x in rdss] + params = {"Namespace":"QCE/CDB","MetricName":i,"Period":60,"StartTime":starttime,"Instances":ins_list} + req.from_json_string(json.dumps(params)) + resp = client.GetMonitorData(req) + metric_list = resp.DataPoints + for metrics in metric_list: + iid = metrics.Dimensions[0].Value + value = metrics.Values[-1] + ts = metrics.Timestamps[-1]*1000 + prom_metric_name = metric_name_dict[i][0].split()[2] + metric_name_dict[i].append(f'{prom_metric_name}{{iid="{iid}"}} {float(value)} {ts}') + prom_metric_list = [] + for x in metric_name_dict.values(): + prom_metric_list = prom_metric_list + x + return prom_metric_list diff --git a/flask-consul/units/prom/redis_ali.py b/flask-consul/units/prom/redis_ali.py index dd61e9a..55e4797 100644 --- a/flask-consul/units/prom/redis_ali.py +++ b/flask-consul/units/prom/redis_ali.py @@ -1,6 +1,4 @@ from aliyunsdkcore.client import AcsClient -from aliyunsdkcore.acs_exception.exceptions import ClientException -from aliyunsdkcore.acs_exception.exceptions import ServerException from aliyunsdkcms.request.v20190101.DescribeMetricLastRequest import DescribeMetricLastRequest from datetime import datetime from units import consul_kv diff --git a/flask-consul/units/prom/redis_huawei.py b/flask-consul/units/prom/redis_huawei.py index e5c0341..3f57aa7 100644 --- a/flask-consul/units/prom/redis_huawei.py +++ b/flask-consul/units/prom/redis_huawei.py @@ -1,6 +1,5 @@ from huaweicloudsdkcore.auth.credentials import BasicCredentials from huaweicloudsdkces.v1.region.ces_region import CesRegion -from huaweicloudsdkcore.exceptions import exceptions from huaweicloudsdkces.v1 import * from datetime import datetime from units import consul_kv @@ -22,33 +21,25 @@ def exporter(vendor,account,region): metric_body_list = [] now = int(datetime.now().timestamp()*1000) redis_list = consul_kv.get_services_list_by_region(f'{vendor}_{account}_redis',region) - try: - for i in metric_name_dict.keys(): - for id in redis_list: - metric_body_list.append(MetricInfo(namespace="SYS.DCS",metric_name=i,dimensions=[MetricsDimension(name="dcs_instance_id",value=id)])) - - request = BatchListMetricDataRequest() - request.body = BatchListMetricDataRequestBody(to=now,_from=now-180000,filter="max",period="1",metrics=metric_body_list) - #print(now-300000,now) - response = client.batch_list_metric_data(request).to_dict() - for i in response['metrics']: - id= i['dimensions'][0]['value'] - try: - value = i['datapoints'][-1]['max'] - ts = i['datapoints'][-1]['timestamp'] - except: - value = -1 - ts = now - metric = i['metric_name'] - prom_metric_name = metric_name_dict[metric][0].split()[2] - metric_name_dict[metric].append(f'{prom_metric_name}{{iid="{id}"}} {float(value)} {ts}') - prom_metric_list = [] - for x in metric_name_dict.values(): - prom_metric_list = prom_metric_list + x - return prom_metric_list - except exceptions.ClientRequestException as e: - print(e.status_code,flush=True) - print(e.request_id,flush=True) - print(e.error_code,flush=True) - print(e.error_msg,flush=True) + for i in metric_name_dict.keys(): + for id in redis_list: + metric_body_list.append(MetricInfo(namespace="SYS.DCS",metric_name=i,dimensions=[MetricsDimension(name="dcs_instance_id",value=id)])) + request = BatchListMetricDataRequest() + request.body = BatchListMetricDataRequestBody(to=now,_from=now-180000,filter="max",period="1",metrics=metric_body_list) + response = client.batch_list_metric_data(request).to_dict() + for i in response['metrics']: + id= i['dimensions'][0]['value'] + try: + value = i['datapoints'][-1]['max'] + ts = i['datapoints'][-1]['timestamp'] + except: + value = -1 + ts = now + metric = i['metric_name'] + prom_metric_name = metric_name_dict[metric][0].split()[2] + metric_name_dict[metric].append(f'{prom_metric_name}{{iid="{id}"}} {float(value)} {ts}') + prom_metric_list = [] + for x in metric_name_dict.values(): + prom_metric_list = prom_metric_list + x + return prom_metric_list diff --git a/flask-consul/units/prom/redis_tencent.py b/flask-consul/units/prom/redis_tencent.py index 0ab2074..c8bdc9a 100644 --- a/flask-consul/units/prom/redis_tencent.py +++ b/flask-consul/units/prom/redis_tencent.py @@ -2,10 +2,10 @@ import json from tencentcloud.common import credential from tencentcloud.common.profile.client_profile import ClientProfile from tencentcloud.common.profile.http_profile import HttpProfile -from tencentcloud.common.exception.tencent_cloud_sdk_exception import TencentCloudSDKException from tencentcloud.monitor.v20180724 import monitor_client, models from datetime import datetime,timedelta from units import consul_kv +from units.config_log import * def exporter(vendor,account,region): ak,sk = consul_kv.get_aksk(vendor,account) @@ -23,27 +23,24 @@ def exporter(vendor,account,region): redis_list = consul_kv.get_services_list_by_region(f'{vendor}_{account}_redis',region) redis_list = list(redis_list) redis_list_10 = [redis_list[i:i + 10] for i in range(0, len(redis_list), 10)] - try: - for i in metric_name_dict.keys(): - for rediss in redis_list_10: - starttime = (datetime.now() + timedelta(minutes=-1)).strftime('%Y-%m-%dT%H:%M:%S+08:00') - ins_list = [{"Dimensions":[{"Name":"instanceid","Value":x}]} for x in rediss] - params = {"Namespace":"QCE/REDIS_MEM","MetricName":i,"Period":60,"StartTime":starttime,"Instances":ins_list} - req.from_json_string(json.dumps(params)) - resp = client.GetMonitorData(req) - metric_list = resp.DataPoints - for metrics in metric_list: - try: - iid = metrics.Dimensions[0].Value - value = metrics.Values[-1] - ts = metrics.Timestamps[-1]*1000 - prom_metric_name = metric_name_dict[i][0].split()[2] - metric_name_dict[i].append(f'{prom_metric_name}{{iid="{iid}"}} {float(value)} {ts}') - except Exception as e: - print("【redis_tencent:prom-metrics-ERROR】",str(e),flush=True) - prom_metric_list = [] - for x in metric_name_dict.values(): - prom_metric_list = prom_metric_list + x - return prom_metric_list - except TencentCloudSDKException as err: - print(err) + for i in metric_name_dict.keys(): + for rediss in redis_list_10: + starttime = (datetime.now() + timedelta(minutes=-1)).strftime('%Y-%m-%dT%H:%M:%S+08:00') + ins_list = [{"Dimensions":[{"Name":"instanceid","Value":x}]} for x in rediss] + params = {"Namespace":"QCE/REDIS_MEM","MetricName":i,"Period":60,"StartTime":starttime,"Instances":ins_list} + req.from_json_string(json.dumps(params)) + resp = client.GetMonitorData(req) + metric_list = resp.DataPoints + for metrics in metric_list: + try: + iid = metrics.Dimensions[0].Value + value = metrics.Values[-1] + ts = metrics.Timestamps[-1]*1000 + prom_metric_name = metric_name_dict[i][0].split()[2] + metric_name_dict[i].append(f'{prom_metric_name}{{iid="{iid}"}} {float(value)} {ts}') + except Exception as e: + logger.error(f"【redis_tencent:prom-metrics-ERROR】{e}") + prom_metric_list = [] + for x in metric_name_dict.values(): + prom_metric_list = prom_metric_list + x + return prom_metric_list diff --git a/flask-consul/units/selfnode_manager.py b/flask-consul/units/selfnode_manager.py index 577c903..2bdeb53 100644 --- a/flask-consul/units/selfnode_manager.py +++ b/flask-consul/units/selfnode_manager.py @@ -2,6 +2,7 @@ import requests,json import sys sys.path.append("..") from config import consul_token,consul_url +from units.config_log import * headers = {'X-Consul-Token': consul_token} @@ -67,6 +68,6 @@ def del_service(vendor,account,region,group,name): if reg.status_code == 200: return {"code": 20000, "data": f"【{sid}】删除成功!"} else: - print(f"{reg.status_code}【{sid}】{reg.text}") + logger.info(f"{reg.status_code}【{sid}】{reg.text}") return {"code": 50000, "data": f"{reg.status_code}【{sid}】{reg.text}"} diff --git a/flask-consul/units/selfrds_manager.py b/flask-consul/units/selfrds_manager.py index d15dae9..9f1a5ac 100644 --- a/flask-consul/units/selfrds_manager.py +++ b/flask-consul/units/selfrds_manager.py @@ -2,7 +2,7 @@ import requests,json import sys sys.path.append("..") from config import consul_token,consul_url - +from units.config_log import * headers = {'X-Consul-Token': consul_token} def get_all_list(vendor,account,region,group): @@ -67,6 +67,6 @@ def del_service(vendor,account,region,group,name): if reg.status_code == 200: return {"code": 20000, "data": f"【{sid}】删除成功!"} else: - print(f"{reg.status_code}【{sid}】{reg.text}") + logger.info(f"{reg.status_code}【{sid}】{reg.text}") return {"code": 50000, "data": f"{reg.status_code}【{sid}】{reg.text}"} diff --git a/flask-consul/units/selfredis_manager.py b/flask-consul/units/selfredis_manager.py index e2fb4c3..32a4fb7 100644 --- a/flask-consul/units/selfredis_manager.py +++ b/flask-consul/units/selfredis_manager.py @@ -2,7 +2,7 @@ import requests,json import sys sys.path.append("..") from config import consul_token,consul_url - +from units.config_log import * headers = {'X-Consul-Token': consul_token} def get_all_list(vendor,account,region,group): @@ -67,6 +67,5 @@ def del_service(vendor,account,region,group,name): if reg.status_code == 200: return {"code": 20000, "data": f"【{sid}】删除成功!"} else: - print(f"{reg.status_code}【{sid}】{reg.text}") + logger.info(f"{reg.status_code}【{sid}】{reg.text}") return {"code": 50000, "data": f"{reg.status_code}【{sid}】{reg.text}"} - diff --git a/flask-consul/units/token_auth.py b/flask-consul/units/token_auth.py index 74791ac..a0bcf2e 100644 --- a/flask-consul/units/token_auth.py +++ b/flask-consul/units/token_auth.py @@ -1,6 +1,7 @@ from flask_httpauth import HTTPTokenAuth from itsdangerous import TimedJSONWebSignatureSerializer from units import consul_kv +from units.config_log import * secret_key = consul_kv.get_value('ConsulManager/assets/secret/skey')['sk'] s = TimedJSONWebSignatureSerializer(secret_key,expires_in=28800) auth = HTTPTokenAuth() @@ -10,7 +11,7 @@ def verify_token(token): try: data = s.loads(token) except Exception as e: - print("【login】认证异常",e,flush=True) + logger.error(f"【login】认证异常,{e}") return False return True diff --git a/flask-consul/units/upload.py b/flask-consul/units/upload.py index 192185d..a376706 100644 --- a/flask-consul/units/upload.py +++ b/flask-consul/units/upload.py @@ -3,6 +3,7 @@ import requests, json import xlrd,re,sys sys.path.append("..") from config import consul_token,consul_url +from units.config_log import * def importconsul(row,imptype): try: @@ -17,7 +18,7 @@ def importconsul(row,imptype): } elif imptype == 'selfnode': vendor,account,region,group,name,instance,os = row - print(row) + logger.info(row) sid = f"{vendor}/{account}/{region}/{group}@{name}" ip = instance.split(':')[0] port = instance.split(':')[1] @@ -33,7 +34,7 @@ def importconsul(row,imptype): } elif imptype == 'selfrds': vendor,account,region,group,name,instance,os = row - print(row) + logger.info(row) sid = f"{vendor}/{account}/{region}/{group}@{name}" ip = instance.split(':')[0] port = instance.split(':')[1] @@ -49,7 +50,7 @@ def importconsul(row,imptype): } elif imptype == 'selfredis': vendor,account,region,group,name,instance,os = row - print(row) + logger.info(row) sid = f"{vendor}/{account}/{region}/{group}@{name}" ip = instance.split(':')[0] port = instance.split(':')[1] @@ -64,22 +65,22 @@ def importconsul(row,imptype): "check": {"tcp": instance,"interval": "60s"} } except Exception as e: - print("【import】导入失败",e,flush=True) + logger.error(f"【import】导入失败,{e}") return {"code": 50000, "data": f"导入内容格式异常!{row}"} headers = {'X-Consul-Token': consul_token} reg = requests.put(f"{consul_url}/agent/service/register", headers=headers, data=json.dumps(data)) if reg.status_code == 200: - print({"code": 20000, "data": "增加成功!"},instance,flush=True) + logger.info(f'code: 20000, data: 增加成功!{instance}') return {"code": 20000, "data": "增加成功!"} else: - print({"code": 50000, "data": f'{reg.status_code}:{reg.text}'},instance,flush=True) + logger.info(f'code: 50000, data: {reg.status_code}:{reg.text},{instance}') return {"code": 50000, "data": f'{reg.status_code}:{reg.text}'} def read_execl(file_contents,imptype): data = xlrd.open_workbook(file_contents=file_contents, encoding_override="utf-8") table = data.sheets()[0] - print("【import】开始读取导入文件",flush=True) + logger.info("【import】开始读取导入文件") for rownum in range(table.nrows): row = table.row_values(rownum) if rownum == 0: diff --git a/flask-consul/views/blackbox.py b/flask-consul/views/blackbox.py index ee07446..70aaec2 100644 --- a/flask-consul/views/blackbox.py +++ b/flask-consul/views/blackbox.py @@ -5,7 +5,7 @@ sys.path.append("..") from units import token_auth,blackbox_manager from werkzeug.datastructures import FileStorage from units import upload - +from units.config_log import * blueprint = Blueprint('blackbox',__name__) api = Api(blueprint) @@ -27,7 +27,7 @@ class Upload(Resource): try: return upload.read_execl(file.read(),'blackbox') except Exception as e: - print("【blackbox】导入失败",e,flush=True) + logger.error(f"【blackbox】导入失败,{e}") return {"code": 50000, "data": f"导入失败!"} class GetAllList(Resource): diff --git a/flask-consul/views/edit_cloud.py b/flask-consul/views/edit_cloud.py index acdbf30..7b9a46f 100644 --- a/flask-consul/views/edit_cloud.py +++ b/flask-consul/views/edit_cloud.py @@ -5,6 +5,7 @@ from config import vendors,regions from units import token_auth,consul_kv from .jobs import deljob,addjob,runjob,modjob_interval,modjob_args import json +from units.config_log import * blueprint = Blueprint('edit_cloud',__name__) api = Api(blueprint) @@ -59,7 +60,7 @@ class Edit(Resource): ecs_interval = int(editjob_dict['ecs_interval']) rds_interval = int(editjob_dict['rds_interval']) redis_interval = int(editjob_dict['redis_interval']) - print(editjob_dict) + logger.info(f'{editjob_dict}') if editjob_dict['akskswitch']: ak = editjob_dict['ak'] sk = editjob_dict['sk'] diff --git a/flask-consul/views/jms.py b/flask-consul/views/jms.py index 14da4bf..a00323f 100644 --- a/flask-consul/views/jms.py +++ b/flask-consul/views/jms.py @@ -5,6 +5,7 @@ from units import token_auth,consul_kv,myaes from config import vendors import json from .jobs import deljob,addjob,runjob,getjob +from units.config_log import * blueprint = Blueprint('jms',__name__) api = Api(blueprint) @@ -105,7 +106,7 @@ class Jms(Resource): custom_ecs_dict = json.loads(custom_ecs_info) consul_kv.put_kv('ConsulManager/jms/custom_ecs_info',custom_ecs_dict) except Exception as e: - print(e,flush=True) + logger.error(f'{e}') return {'code': 50000, 'data': 'Json解析错误,请检查!'} else: consul_kv.put_kv('ConsulManager/jms/custom_ecs_info',{}) diff --git a/flask-consul/views/login.py b/flask-consul/views/login.py index b806e12..0a93002 100644 --- a/flask-consul/views/login.py +++ b/flask-consul/views/login.py @@ -6,6 +6,7 @@ sys.path.append("..") from config import admin_passwd from units import token_auth, consul_kv from units.ldap.LdapUser import Ldap +from units.config_log import * secret_key = consul_kv.get_value('ConsulManager/assets/secret/skey')['sk'] s = TimedJSONWebSignatureSerializer(secret_key,expires_in=28800) @@ -32,7 +33,7 @@ class User(Resource): ldap = False if username == 'admin' else 'True' #ldap认证 if user_opt == 'login' and ldap == "True": - print("ldap") + logger.info("ldap") ldap_obj = Ldap() ldap_result = ldap_obj.authpass(username,password) if ldap_result == 1: @@ -46,7 +47,7 @@ class User(Resource): return {"code": 40000, "data": "LDAP用户密码错误!"} else: if user_opt == 'login': - print("非ldap") + logger.info("非ldap") if password == admin_passwd: token = str(s.dumps(admin_passwd),encoding="utf-8") return {"code": 20000,"data": {"token": "Bearer " + token,"username":username}} diff --git a/flask-consul/views/nodes.py b/flask-consul/views/nodes.py index 1c16ef2..b714b4a 100644 --- a/flask-consul/views/nodes.py +++ b/flask-consul/views/nodes.py @@ -4,7 +4,7 @@ from flask_apscheduler import APScheduler #import sys #sys.path.append("..") from units import token_auth,consul_kv,gen_config,consul_svc - +from units.config_log import * blueprint = Blueprint('nodes',__name__) api = Api(blueprint) @@ -97,7 +97,7 @@ class Nodes(Resource): consul_svc.add_sid(sid_dict) return {'code': 20000, 'data': '自定义实例信息修改成功!'} except Exception as e: - print(e,flush=True) + logger.error(f'{e}') return {'code': 50000, "data": '提交自定义实例信息格式错误!'} diff --git a/flask-consul/views/rds.py b/flask-consul/views/rds.py index 8cec3ec..2546155 100644 --- a/flask-consul/views/rds.py +++ b/flask-consul/views/rds.py @@ -4,7 +4,7 @@ from flask_apscheduler import APScheduler #import sys #sys.path.append("..") from units import token_auth,consul_kv,gen_config,consul_svc - +from units.config_log import * blueprint = Blueprint('rds',__name__) api = Api(blueprint) @@ -86,7 +86,7 @@ class Rds(Resource): consul_svc.add_sid(sid_dict) return {'code': 20000, 'data': '自定义实例信息修改成功!'} except Exception as e: - print(e,flush=True) + logger.error(f'{e}') return {'code': 50000, "data": '提交自定义实例信息格式错误!'} api.add_resource(Rds, '/api/rds/') diff --git a/flask-consul/views/redis.py b/flask-consul/views/redis.py index 3014f2f..e87b6de 100644 --- a/flask-consul/views/redis.py +++ b/flask-consul/views/redis.py @@ -4,7 +4,7 @@ from flask_apscheduler import APScheduler #import sys #sys.path.append("..") from units import token_auth,consul_kv,gen_config,consul_svc - +from units.config_log import * blueprint = Blueprint('redis',__name__) api = Api(blueprint) @@ -86,7 +86,7 @@ class Redis(Resource): consul_svc.add_sid(sid_dict) return {'code': 20000, 'data': '自定义实例信息修改成功!'} except Exception as e: - print(e,flush=True) + logger.error(f'{e}') return {'code': 50000, "data": '提交自定义实例信息格式错误!'} api.add_resource(Redis, '/api/redis/') diff --git a/flask-consul/views/selfnode.py b/flask-consul/views/selfnode.py index 5435e89..b5e6bc3 100644 --- a/flask-consul/views/selfnode.py +++ b/flask-consul/views/selfnode.py @@ -5,7 +5,7 @@ sys.path.append("..") from units import token_auth,selfnode_manager from werkzeug.datastructures import FileStorage from units import upload - +from units.config_log import * blueprint = Blueprint('selfnode',__name__) api = Api(blueprint) @@ -29,7 +29,7 @@ class Upload(Resource): try: return upload.read_execl(file.read(),'selfnode') except Exception as e: - print("【selfnode】导入失败",e,flush=True) + logger.error(f"【selfnode】导入失败,{e}") return {"code": 50000, "data": f"导入失败!"} class GetAllList(Resource): @@ -44,7 +44,7 @@ class SelfnodeApi(Resource): return selfnode_manager.get_service() def post(self): args = parser.parse_args() - print('=======\n',args,flush=True) + logger.info(f'=======\n,{args}') return selfnode_manager.add_service(args['vendor'],args['account'],args['region'], args['group'],args['name'],args['ip'],args['port'],args['os']) def put(self): diff --git a/flask-consul/views/selfrds.py b/flask-consul/views/selfrds.py index 2247228..c706bf1 100644 --- a/flask-consul/views/selfrds.py +++ b/flask-consul/views/selfrds.py @@ -5,7 +5,7 @@ sys.path.append("..") from units import token_auth,selfrds_manager from werkzeug.datastructures import FileStorage from units import upload - +from units.config_log import * blueprint = Blueprint('selfrds',__name__) api = Api(blueprint) @@ -29,7 +29,7 @@ class Upload(Resource): try: return upload.read_execl(file.read(),'selfrds') except Exception as e: - print("【selfrds】导入失败",e,flush=True) + logger.error(f"【selfrds】导入失败,{e}") return {"code": 50000, "data": f"导入失败!"} class GetAllList(Resource): @@ -44,7 +44,7 @@ class SelfrdsApi(Resource): return selfrds_manager.get_service() def post(self): args = parser.parse_args() - print('=======\n',args,flush=True) + logger.info(f'=======\n{args}') return selfrds_manager.add_service(args['vendor'],args['account'],args['region'], args['group'],args['name'],args['ip'],args['port'],args['os']) def put(self): diff --git a/flask-consul/views/selfredis.py b/flask-consul/views/selfredis.py index 43460e2..56da588 100644 --- a/flask-consul/views/selfredis.py +++ b/flask-consul/views/selfredis.py @@ -5,7 +5,7 @@ sys.path.append("..") from units import token_auth,selfredis_manager from werkzeug.datastructures import FileStorage from units import upload - +from units.config_log import * blueprint = Blueprint('selfredis',__name__) api = Api(blueprint) @@ -29,7 +29,7 @@ class Upload(Resource): try: return upload.read_execl(file.read(),'selfredis') except Exception as e: - print("【selfredis】导入失败",e,flush=True) + logger.error(f"【selfredis】导入失败,{e}") return {"code": 50000, "data": f"导入失败!"} class GetAllList(Resource): @@ -44,7 +44,7 @@ class SelfredisApi(Resource): return selfredis_manager.get_service() def post(self): args = parser.parse_args() - print('=======\n',args,flush=True) + logger.info(f'=======\n,{args}') return selfredis_manager.add_service(args['vendor'],args['account'],args['region'], args['group'],args['name'],args['ip'],args['port'],args['os']) def put(self):