github
/
InternLM
mirror of https://github.com/InternLM/InternLM
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

init light monitoring on all ranks (#462)

pull/486/head
jiaopenglong 2023-11-09 20:04:21 +08:00 committed by GitHub
parent 0218e3131c
commit 0763bf3972
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 4 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
internlm/initialize/launch.py
View File

@ -532,11 +532,11 @@ def initialize_distributed_env(
# init light monitor client # init light monitor client
if gpc.config.get("monitor") and gpc.config.monitor.get("alert"): if gpc.config.get("monitor") and gpc.config.monitor.get("alert"):
alert_config = gpc.config.monitor.alert alert_config = gpc.config.monitor.alert
if alert_config.enable_feishu_alert and gpc.is_rank_for_log(): if alert_config.enable_feishu_alert:
light_monitor_address = alert_config.light_monitor_address light_monitor_address = alert_config.light_monitor_address
if light_monitor_address: if light_monitor_address:
initialize_light_monitor(light_monitor_address) initialize_light_monitor(light_monitor_address)
else: elif gpc.is_rank_for_log():
logger.warning("monitor address is none, monitor could not be used!") logger.warning("monitor address is none, monitor could not be used!")

3
internlm/monitor/alert.py
View File

@ -7,6 +7,7 @@ from typing import Dict
import requests import requests
from internlm.core.context import global_context as gpc
from internlm.utils.logger import get_logger from internlm.utils.logger import get_logger
logger = get_logger(__file__) logger = get_logger(__file__)
@ -29,7 +30,7 @@ def initialize_light_monitor(monitor_address: str = None):
try: try:
from uniscale_monitoring import init_monitor from uniscale_monitoring import init_monitor
init_monitor(monitor_address) init_monitor(monitor_address, is_root_rank=gpc.is_rank_for_log())
except Exception as e: except Exception as e:
logger.warning(f"init monitor meet error: {e}") logger.warning(f"init monitor meet error: {e}")