diff --git a/ptuning/main.py b/ptuning/main.py
index 17e18b5..ff01de2 100644
--- a/ptuning/main.py
+++ b/ptuning/main.py
@@ -47,7 +47,7 @@ from arguments import ModelArguments, DataTrainingArguments
 logger = logging.getLogger(__name__)
 
 def main():
-
+    torch.distributed.init_process_group(backend='nccl')
     parser = HfArgumentParser((ModelArguments, DataTrainingArguments, Seq2SeqTrainingArguments))
     if len(sys.argv) == 2 and sys.argv[1].endswith(".json"):
         # If we pass only one argument to the script and it's the path to a json file,