Update test_ci.sh

update

Update test_ci.sh

Update test_ci.sh

Update test_ci.sh

Update test_ci.sh

Update test_ci.sh

Update test_ci.sh

Update run_chatgpt_examples.yml

Update run_chatgpt_examples.yml

Update run_chatgpt_examples.yml

Update run_chatgpt_examples.yml

Update run_chatgpt_examples.yml

Update run_chatgpt_examples.yml

Update test_ci.sh

Update test_ci.sh

update

Update run_chatgpt_examples.yml

Update run_chatgpt_examples.yml

update ci

Update test_ci.sh

Update run_chatgpt_examples.yml

Update run_chatgpt_examples.yml

Update run_chatgpt_examples.yml

Update run_chatgpt_examples.yml

Update run_chatgpt_examples.yml

Update run_chatgpt_examples.yml

Update run_chatgpt_examples.yml

Update test_ci.sh

Update test_ci.sh

Update run_chatgpt_examples.yml

Update test_ci.sh

Update test_ci.sh

Update test_ci.sh

update test ci

RoBERTa for RLHF Stage 2 & 3 (still in testing)

Revert "Add RoBERTa for RLHF Stage 2 & 3 (test)"

This reverts commit 06741d894d.

Add RoBERTa for RLHF stage 2 & 3

1. add roberta folder under model folder
2. add  roberta option in train_reward_model.py
3. add some test in testci

Update test_ci.sh

Revert "Update test_ci.sh"

This reverts commit 9c7352b81766f3177d31eeec0ec178a301df966a.

Add RoBERTa for RLHF Stage 2 & 3 (test)

RoBERTa for RLHF Stage 2 & 3 (still in testing)

Revert "Add RoBERTa for RLHF Stage 2 & 3 (test)"

This reverts commit 06741d894d.

Add RoBERTa for RLHF stage 2 & 3

1. add roberta folder under model folder
2. add  roberta option in train_reward_model.py
3. add some test in testci

Update test_ci.sh

Revert "Update test_ci.sh"

This reverts commit 9c7352b81766f3177d31eeec0ec178a301df966a.

update roberta with coati

chat ci update

Revert "chat ci update"

This reverts commit 17ae7ae01fa752bd3289fc39069868fde99cf846.

[test]chat_update_ci

Update test_ci.sh

Update test_ci.sh

test

Update gpt_critic.py

Update gpt_critic.py

Update run_chatgpt_unit_tests.yml

update test ci

update

update

update

update

Update test_ci.sh

update

Update test_ci.sh

Update test_ci.sh

Update run_chatgpt_examples.yml

Update run_chatgpt_examples.yml

.github/workflows/run_chatgpt_examples.yml

@@ -4,10 +4,10 @@ on:
   pull_request:
     types: [synchronize, opened, reopened]
     paths:
-      - 'applications/ChatGPT/chatgpt/**'
+      - 'applications/Chat/coati/**'
-      - 'applications/ChatGPT/requirements.txt'
+      - 'applications/Chat/requirements.txt'
-      - 'applications/ChatGPT/setup.py'
+      - 'applications/Chat/setup.py'
-      - 'applications/ChatGPT/examples/**'
+      - 'applications/Chat/examples/**'
 
 
 jobs:
@@ -16,7 +16,7 @@ jobs:
     runs-on: [self-hosted, gpu]
     container:
       image: hpcaitech/pytorch-cuda:1.12.0-11.3.0
-      options: --gpus all --rm -v /data/scratch/chatgpt:/data/scratch/chatgpt
+      options: --gpus all --rm -v /data/scratch/github_actions/chat:/data/scratch/github_actions/chat
     timeout-minutes: 30
     defaults:
       run:
@@ -27,17 +27,26 @@ jobs:
 
       - name: Install ColossalAI and ChatGPT
         run: |
-          pip install -v .
+          pip install -e .
-          cd applications/ChatGPT
+          cd applications/Chat
           pip install -v .
           pip install -r examples/requirements.txt
 
+      - name: Install Transformers
+        run: |
+          cd applications/Chat
+          git clone https://github.com/hpcaitech/transformers
+          cd transformers
+          pip install -v .
+
       - name: Execute Examples
         run: |
-          cd applications/ChatGPT
+          cd applications/Chat
           rm -rf ~/.cache/colossalai
           ./examples/test_ci.sh
         env:
           NCCL_SHM_DISABLE: 1
           MAX_JOBS: 8
-          PROMPT_PATH: /data/scratch/chatgpt/prompts.csv
+          SFT_DATASET: /data/scratch/github_actions/chat/data.json
+          PROMPT_PATH: /data/scratch/github_actions/chat/prompts_en.jsonl
+          PRETRAIN_DATASET: /data/scratch/github_actions/chat/alpaca_data.json

applications/Chat/coati/models/gpt/gpt_actor.py

@@ -23,7 +23,8 @@ class GPTActor(Actor):
                  config: Optional[GPT2Config] = None,
                  checkpoint: bool = False,
                  lora_rank: int = 0,
-                 lora_train_bias: str = 'none') -> None:
+                 lora_train_bias: str = 'none',
+                 **kwargs) -> None:
         if pretrained is not None:
             model = GPT2LMHeadModel.from_pretrained(pretrained)
         elif config is not None:
@@ -32,4 +33,4 @@ class GPTActor(Actor):
             model = GPT2LMHeadModel(GPT2Config())
         if checkpoint:
             model.gradient_checkpointing_enable()
-        super().__init__(model, lora_rank, lora_train_bias)
+        super().__init__(model, lora_rank, lora_train_bias, **kwargs)

applications/Chat/coati/models/gpt/gpt_critic.py

@@ -24,7 +24,8 @@ class GPTCritic(Critic):
                  config: Optional[GPT2Config] = None,
                  checkpoint: bool = False,
                  lora_rank: int = 0,
-                 lora_train_bias: str = 'none') -> None:
+                 lora_train_bias: str = 'none',
+                 **kwargs) -> None:
         if pretrained is not None:
             model = GPT2Model.from_pretrained(pretrained)
         elif config is not None:
@@ -34,4 +35,4 @@ class GPTCritic(Critic):
         if checkpoint:
             model.gradient_checkpointing_enable()
         value_head = nn.Linear(model.config.n_embd, 1)
-        super().__init__(model, value_head, lora_rank, lora_train_bias)
+        super().__init__(model, value_head, lora_rank, lora_train_bias, **kwargs)

applications/Chat/examples/test_ci.sh

@@ -2,11 +2,21 @@
 
 set -xue
 
+if [ -z "$SFT_DATASET" ]; then
+    echo "Please set \$SFT_DATASET to the path to sft dataset."
+    exit 1
+fi
+
 if [ -z "$PROMPT_PATH" ]; then
     echo "Please set \$PROMPT_PATH to the path to prompts csv."
     exit 1
 fi
 
+if [ -z "$PRETRAIN_DATASET" ]; then
+    echo "Please set \$PRETRAIN_DATASET to the path to alpaca data."
+    exit 1
+fi
+
 BASE=$(realpath $(dirname $0))
 
 export OMP_NUM_THREADS=8
@@ -14,104 +24,97 @@ export OMP_NUM_THREADS=8
 # install requirements
 pip install -r ${BASE}/requirements.txt
 
-# train dummy
+wandb init -m offline
-python ${BASE}/train_dummy.py --strategy naive --num_episodes 1 \
-                              --max_timesteps 2 --update_timesteps 2 \
-                              --max_epochs 1 --train_batch_size 2 --lora_rank 4
 
-torchrun --standalone --nproc_per_node=2 ${BASE}/train_dummy.py \
+# train sft
-         --strategy colossalai_gemini --num_episodes 1 --max_timesteps 2 \
+torchrun --standalone --nproc_per_node=4 ${BASE}/train_sft.py --pretrain 'bigscience/bloom-560m' \
-         --update_timesteps 2 --max_epochs 1 --train_batch_size 2\
+        --model 'bloom' --strategy colossalai_zero2 --lora_rank 4\
-         --pretrain 'facebook/opt-350m' --model opt --lora_rank 4\
+        --dataset $SFT_DATASET --max_datasets_size 512 --max_epochs 1 \
-         --save_path ${BASE}/actor_checkpoint_dummy.pt
+        --save_path ${BASE}/output
-python ${BASE}/inference.py --model_path ${BASE}/actor_checkpoint_dummy.pt --pretrain 'facebook/opt-350m' --model opt
 
-torchrun --standalone --nproc_per_node=2 ${BASE}/train_dummy.py \
+torchrun --standalone --nproc_per_node=4 ${BASE}/train_sft.py --pretrain 'gpt2' \
-         --strategy ddp --num_episodes 1 --max_timesteps 2 \
+        --model 'gpt2' --strategy colossalai_zero2 \
-         --update_timesteps 2 --max_epochs 1 --train_batch_size 2\
+        --dataset $SFT_DATASET --max_datasets_size 512 --max_epochs 1 \
-         --pretrain 'facebook/opt-350m' --model opt --lora_rank 4\
+        --save_path ${BASE}/output
-         --save_path ${BASE}/actor_checkpoint_dummy.pt
-python ${BASE}/inference.py --model_path ${BASE}/actor_checkpoint_dummy.pt --pretrain 'facebook/opt-350m' --model opt
 
-torchrun --standalone --nproc_per_node=2 ${BASE}/train_dummy.py \
+torchrun --standalone --nproc_per_node=4 ${BASE}/train_sft.py --pretrain 'facebook/opt-350m' \
-         --strategy colossalai_zero2 --num_episodes 1 --max_timesteps 2 \
+        --model 'opt' --strategy colossalai_zero2 --lora_rank 4\
-         --update_timesteps 2 --max_epochs 1 --train_batch_size 2\
+        --dataset $SFT_DATASET --max_datasets_size 512 --max_epochs 1 \
-         --pretrain 'gpt2' --model gpt2 --lora_rank 4\
+        --save_path ${BASE}/output
-         --save_path ${BASE}/actor_checkpoint_dummy.pt
-python ${BASE}/inference.py --model_path ${BASE}/actor_checkpoint_dummy.pt --pretrain 'gpt2' --model gpt2
 
-torchrun --standalone --nproc_per_node=2 ${BASE}/train_dummy.py \
+torchrun --standalone --nproc_per_node=4 ${BASE}/train_sft.py --pretrain 'gpt2' \
-         --strategy colossalai_zero2 --num_episodes 1 --max_timesteps 2 \
+        --model 'gpt2' --strategy ddp --lora_rank 4\
-         --update_timesteps 2 --max_epochs 1 --train_batch_size 2\
+        --dataset $SFT_DATASET --max_datasets_size 512 --max_epochs 1 \
-         --pretrain 'roberta-base' --model roberta --lora_rank 4\
+        --save_path ${BASE}/output
-         --save_path ${BASE}/actor_checkpoint_dummy.pt
-python ${BASE}/inference.py --model_path ${BASE}/actor_checkpoint_dummy.pt --pretrain 'roberta-base' --model roberta
 
-rm -rf ${BASE}/actor_checkpoint_dummy.pt
+#torchrun --standalone --nproc_per_node=4 ${BASE}/train_sft.py --pretrain 'facebook/opt-350m' \
+#        --model 'opt' --strategy naive \
+#        --dataset $SFT_DATASET --max_datasets_size 512 --max_epochs 1 \
+#        --save_path ${BASE}/output
 
-# train prompts
+rm -rf ${BASE}/output
-python ${BASE}/train_prompts.py $PROMPT_PATH --strategy naive --num_episodes 1 \
-                                             --max_timesteps 2 --update_timesteps 2 \
-                                             --max_epochs 1 --train_batch_size 2 --lora_rank 4
-
-torchrun --standalone --nproc_per_node=2 ${BASE}/train_prompts.py $PROMPT_PATH \
-         --strategy colossalai_zero2 --num_episodes 1 --max_timesteps 2 \
-         --update_timesteps 2 --max_epochs 1 --train_batch_size 2\
-         --pretrain 'facebook/opt-350m' --model opt --lora_rank 4\
-         --save_path ${BASE}/actor_checkpoint_prompts.pt
-python ${BASE}/inference.py --model_path ${BASE}/actor_checkpoint_prompts.pt --pretrain 'facebook/opt-350m' --model opt
-
-torchrun --standalone --nproc_per_node=2 ${BASE}/train_prompts.py $PROMPT_PATH \
-         --strategy ddp --num_episodes 1 --max_timesteps 2 \
-         --update_timesteps 2 --max_epochs 1 --train_batch_size 2\
-         --pretrain 'gpt2' --model gpt2 --lora_rank 4\
-         --save_path ${BASE}/actor_checkpoint_prompts.pt
-python ${BASE}/inference.py --model_path ${BASE}/actor_checkpoint_prompts.pt --pretrain 'gpt2' --model gpt2
-
-torchrun --standalone --nproc_per_node=2 ${BASE}/train_prompts.py $PROMPT_PATH \
-         --strategy colossalai_gemini --num_episodes 1 --max_timesteps 2 \
-         --update_timesteps 2 --max_epochs 1 --train_batch_size 2\
-         --pretrain 'gpt2' --model gpt2 --lora_rank 4\
-         --save_path ${BASE}/actor_checkpoint_prompts.pt
-python ${BASE}/inference.py --model_path ${BASE}/actor_checkpoint_prompts.pt --pretrain 'gpt2' --model gpt2
-
-torchrun --standalone --nproc_per_node=2 ${BASE}/train_prompts.py $PROMPT_PATH \
-         --strategy colossalai_zero2 --num_episodes 1 --max_timesteps 2 \
-         --update_timesteps 2 --max_epochs 1 --train_batch_size 2\
-         --pretrain 'roberta-base' --model roberta --lora_rank 4\
-         --save_path ${BASE}/actor_checkpoint_prompts.pt
-python ${BASE}/inference.py --model_path ${BASE}/actor_checkpoint_prompts.pt --pretrain 'roberta-base' --model roberta
-
-rm -rf ${BASE}/actor_checkpoint_prompts.pt
 
 # train rm
 torchrun --standalone --nproc_per_node=2 ${BASE}/train_reward_model.py \
-                             --pretrain 'facebook/opt-350m' --model 'opt' \
+                            --pretrain 'facebook/opt-350m' --model 'opt' \
-                             --strategy colossalai_zero2 --loss_fn 'log_sig'\
+                            --strategy colossalai_zero2 --loss_fn 'log_sig'\
-                             --dataset 'Anthropic/hh-rlhf' --subset 'harmless-base'\
+                            --dataset 'Anthropic/hh-rlhf' --subset 'harmless-base' \
-                             --test True --lora_rank 4
+                            --test True --lora_rank 4 \
+                            --save_path ${BASE}/rm_ckpt_opt.pt
 
 torchrun --standalone --nproc_per_node=2 ${BASE}/train_reward_model.py \
-                             --pretrain 'gpt2' --model 'gpt2' \
+                            --pretrain 'gpt2' --model 'gpt2' \
-                             --strategy colossalai_gemini --loss_fn 'log_exp'\
+                            --strategy colossalai_zero2 --loss_fn 'log_exp' \
-                             --dataset 'Dahoas/rm-static' --test True --lora_rank 4
+                            --dataset 'Dahoas/rm-static' \
+                            --test True  --lora_rank 4 \
+                            --save_path ${BASE}/rm_ckpt_gpt.pt
 
 torchrun --standalone --nproc_per_node=2 ${BASE}/train_reward_model.py \
-                             --pretrain 'bigscience/bloom-560m' --model 'bloom' \
+                            --pretrain 'gpt2' --model 'gpt2' \
-                             --strategy colossalai_zero2 --loss_fn 'log_sig'\
+                            --strategy ddp --loss_fn 'log_exp' \
-                             --dataset 'Anthropic/hh-rlhf' --subset 'harmless-base'\
+                            --dataset 'Dahoas/rm-static' \
-                             --test True --lora_rank 4
+                            --test True --lora_rank 4 \
+                            --save_path ${BASE}/rm_ckpt.pt
 
 torchrun --standalone --nproc_per_node=2 ${BASE}/train_reward_model.py \
-                             --pretrain 'microsoft/deberta-v3-large' --model 'deberta' \
+                            --pretrain 'bigscience/bloom-560m' --model 'bloom' \
-                             --strategy colossalai_zero2 --loss_fn 'log_sig'\
+                            --strategy colossalai_zero2 --loss_fn 'log_sig' \
-                             --dataset 'Anthropic/hh-rlhf' --subset 'harmless-base'\
+                            --dataset 'Anthropic/hh-rlhf' --subset 'harmless-base' \
-                             --test True --lora_rank 4
+                            --test True --lora_rank 4 \
+                            --save_path ${BASE}/rm_ckpt.pt
 
 torchrun --standalone --nproc_per_node=2 ${BASE}/train_reward_model.py \
-                             --pretrain 'roberta-base' --model 'roberta' \
+                            --pretrain 'microsoft/deberta-v3-large' --model 'deberta' \
-                             --strategy colossalai_zero2 --loss_fn 'log_exp'\
+                            --strategy colossalai_zero2 --loss_fn 'log_sig' \
-                             --dataset 'Anthropic/hh-rlhf' --subset 'harmless-base'\
+                            --dataset 'Anthropic/hh-rlhf' --subset 'harmless-base' \
-                             --test True --lora_rank 4
+                            --test True --lora_rank 4 \
+                            --save_path ${BASE}/rm_ckpt.pt
+
+torchrun --standalone --nproc_per_node=2 ${BASE}/train_reward_model.py \
+                            --pretrain 'roberta-base' --model 'roberta' \
+                            --strategy colossalai_zero2 --loss_fn 'log_exp'\
+                            --dataset 'Anthropic/hh-rlhf' --subset 'harmless-base'\
+                            --test True --lora_rank 4 \
+                            --save_path ${BASE}/rm_ckpt.pt
 
 rm -rf ${BASE}/rm_ckpt.pt
+
+torchrun --standalone --nproc_per_node=2 ${BASE}/train_prompts.py --prompt_path $PROMPT_PATH --pretrain_dataset $PRETRAIN_DATASET \
+        --strategy colossalai_zero2 --num_episodes 1 --max_timesteps 2 \
+        --update_timesteps 2 --max_epochs 1 --train_batch_size 2 \
+        --pretrain 'facebook/opt-350m' --model opt \
+        --rm_pretrain 'facebook/opt-350m' \
+        --rm_path ${BASE}/rm_ckpt_opt.pt \
+        --save_path ${BASE}/actor_checkpoint_prompts.pt
+rm -rf ${BASE}/rm_ckpt_opt.pt
+
+torchrun --standalone --nproc_per_node=2 ${BASE}/train_prompts.py --prompt_path $PROMPT_PATH --pretrain_dataset $PRETRAIN_DATASET \
+         --strategy colossalai_zero2 --num_episodes 1 --max_timesteps 2 \
+         --update_timesteps 2 --max_epochs 1 --train_batch_size 2 \
+         --pretrain 'gpt2' --model gpt2 \
+         --rm_pretrain 'gpt2' \
+         --rm_path ${BASE}/rm_ckpt_gpt.pt \
+         --save_path ${BASE}/actor_checkpoint_prompts.pt
+rm -rf ${BASE}/rm_ckpt_gpt.pt
+
+rm -rf ${BASE}/actor_checkpoint_prompts.pt