github
/
InternLM
mirror of https://github.com/InternLM/InternLM
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

fix(utils): disable bench_net in gputest.py (#421)

pull/424/head
Guoteng 2023-10-19 10:00:57 +08:00 committed by GitHub
parent 4b5bdedff2
commit 3ea94f2e2a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 3 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
internlm/utils/gputest.py
View File

@ -39,7 +39,9 @@ def empty_cache_and_diag(batch_count, interval=50):
with torch.no_grad(): with torch.no_grad():
timer_diagnosis() timer_diagnosis()
bench_gpu() bench_gpu()
bench_net() # FIXME: Runtime benchmark diagnosis can easily cause the training process
# to exit due to NCCL errors.
# bench_net()
# do empty_cache after the bench # do empty_cache after the bench
torch.cuda.empty_cache() torch.cuda.empty_cache()
# do garbage collection # do garbage collection