github
/
ColossalAI
mirror of https://github.com/hpcaitech/ColossalAI
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

fix wrong EOS token in ColossalChat

pull/5060/head
Orion-Zheng 2023-11-14 09:58:00 +08:00 committed by Zian(Andy) Zheng
parent 70885d707d
commit 43ad0d9ef0
6 changed files with 6 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
applications/Chat/examples/community/peft/train_peft_prompts.py
View File

@ -118,7 +118,7 @@ def main(args):
tokenizer.pad_token = tokenizer.eos_token tokenizer.pad_token = tokenizer.eos_token
elif args.model == "llama": elif args.model == "llama":
tokenizer = LlamaTokenizer.from_pretrained(args.pretrain) tokenizer = LlamaTokenizer.from_pretrained(args.pretrain)
tokenizer.eos_token = "<\s>" tokenizer.eos_token = "</s>"
tokenizer.pad_token = tokenizer.unk_token tokenizer.pad_token = tokenizer.unk_token
else: else:
raise ValueError(f'Unsupported model "{args.model}"') raise ValueError(f'Unsupported model "{args.model}"')

2
applications/Chat/examples/community/peft/train_peft_sft.py
View File

@ -68,7 +68,7 @@ def train(args):
padding_side="right", padding_side="right",
use_fast=False, use_fast=False,
) )
tokenizer.eos_token = "<\s>" tokenizer.eos_token = "</s>"
tokenizer.pad_token = tokenizer.unk_token tokenizer.pad_token = tokenizer.unk_token
else: else:
raise ValueError(f'Unsupported model "{args.model}"') raise ValueError(f'Unsupported model "{args.model}"')

2
applications/Chat/examples/inference.py
View File

@ -39,7 +39,7 @@ def eval(args):
tokenizer.pad_token = tokenizer.eos_token tokenizer.pad_token = tokenizer.eos_token
elif args.model == "llama": elif args.model == "llama":
tokenizer = LlamaTokenizer.from_pretrained("hf-internal-testing/llama-tokenizer") tokenizer = LlamaTokenizer.from_pretrained("hf-internal-testing/llama-tokenizer")
tokenizer.eos_token = "<\s>" tokenizer.eos_token = "</s>"
tokenizer.pad_token = tokenizer.unk_token tokenizer.pad_token = tokenizer.unk_token
else: else:
raise ValueError(f'Unsupported model "{args.model}"') raise ValueError(f'Unsupported model "{args.model}"')

2
applications/Chat/examples/train_prompts.py
View File

@ -125,7 +125,7 @@ def main(args):
tokenizer = LlamaTokenizer.from_pretrained( tokenizer = LlamaTokenizer.from_pretrained(
"hf-internal-testing/llama-tokenizer" if args.tokenizer is None else args.tokenizer "hf-internal-testing/llama-tokenizer" if args.tokenizer is None else args.tokenizer
) )
tokenizer.eos_token = "<\s>" tokenizer.eos_token = "</s>"
tokenizer.pad_token = tokenizer.unk_token tokenizer.pad_token = tokenizer.unk_token
else: else:
raise ValueError(f'Unsupported model "{args.model}"') raise ValueError(f'Unsupported model "{args.model}"')

2
applications/Chat/examples/train_reward_model.py
View File

@ -72,7 +72,7 @@ def train(args):
tokenizer = LlamaTokenizer.from_pretrained( tokenizer = LlamaTokenizer.from_pretrained(
"hf-internal-testing/llama-tokenizer" if args.tokenizer is None else args.tokenizer "hf-internal-testing/llama-tokenizer" if args.tokenizer is None else args.tokenizer
) )
tokenizer.eos_token = "<\s>" tokenizer.eos_token = "</s>"
tokenizer.pad_token = tokenizer.unk_token tokenizer.pad_token = tokenizer.unk_token
else: else:
raise ValueError(f'Unsupported model "{args.model}"') raise ValueError(f'Unsupported model "{args.model}"')

2
applications/Chat/examples/train_sft.py
View File

@ -75,7 +75,7 @@ def train(args):
tokenizer = LlamaTokenizer.from_pretrained( tokenizer = LlamaTokenizer.from_pretrained(
"hf-internal-testing/llama-tokenizer" if args.tokenizer is None else args.tokenizer "hf-internal-testing/llama-tokenizer" if args.tokenizer is None else args.tokenizer
) )
tokenizer.eos_token = "<\s>" tokenizer.eos_token = "</s>"
tokenizer.pad_token = tokenizer.unk_token tokenizer.pad_token = tokenizer.unk_token
elif args.model == "chatglm": elif args.model == "chatglm":
tokenizer = ChatGLMTokenizer.from_pretrained( tokenizer = ChatGLMTokenizer.from_pretrained(