fix(utils): disable bench_net in gputest.py

pull/421/head
877825076@qq.com 2023-10-18 12:26:24 +08:00
parent 9a731b6e9b
commit e3d128230b
1 changed files with 3 additions and 1 deletions

View File

@ -39,7 +39,9 @@ def empty_cache_and_diag(batch_count, interval=50):
with torch.no_grad():
timer_diagnosis()
bench_gpu()
bench_net()
# FIXME: Runtime benchmark diagnosis can easily cause the training process
# to exit due to NCCL errors.
# bench_net()
# do empty_cache after the bench
torch.cuda.empty_cache()
# do garbage collection