fix(utils): disable bench_net in gputest.py (#421)

pull/424/head
Guoteng 2023-10-19 10:00:57 +08:00 committed by GitHub
parent 4b5bdedff2
commit 3ea94f2e2a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 3 additions and 1 deletions

View File

@ -39,7 +39,9 @@ def empty_cache_and_diag(batch_count, interval=50):
with torch.no_grad():
timer_diagnosis()
bench_gpu()
bench_net()
# FIXME: Runtime benchmark diagnosis can easily cause the training process
# to exit due to NCCL errors.
# bench_net()
# do empty_cache after the bench
torch.cuda.empty_cache()
# do garbage collection