mirror of https://github.com/hpcaitech/ColossalAI
polish
parent
9ae9e74017
commit
ad00894f7f
|
@ -1,4 +1,4 @@
|
|||
# Auto-Parallelism with GPT2
|
||||
# Pipeline Parallelism Demo with GPT2
|
||||
|
||||
## Requirements
|
||||
|
||||
|
@ -33,5 +33,6 @@ For simplicity, the input data is randonly generated here.
|
|||
|
||||
```bash
|
||||
#Run the Pipeline Parallel on GPT with default setting and a dummy dataset.
|
||||
#You can change the GPU number or microbatch number in the run.sh .
|
||||
bash run.sh
|
||||
```
|
||||
|
|
|
@ -1,12 +0,0 @@
|
|||
import torch
|
||||
|
||||
|
||||
# Randomly Generated Data
|
||||
def get_data(batch_size, seq_len, vocab_size):
|
||||
input_ids = torch.randint(0, vocab_size, (batch_size, seq_len), device=torch.cuda.current_device())
|
||||
attention_mask = torch.ones_like(input_ids)
|
||||
return input_ids, attention_mask
|
||||
|
||||
|
||||
def get_tflops(model_numel, batch_size, seq_len, step_time):
|
||||
return model_numel * batch_size * seq_len * 8 / 1e12 / (step_time + 1e-12)
|
Loading…
Reference in New Issue