github
/
InternLM
mirror of https://github.com/InternLM/InternLM
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

set pos eos pad in convert2hf to fix improper generation

pull/471/head
x54-729 2023-11-03 22:30:23 +08:00
parent 42dfbbebb3
commit 3a1dd36d05
1 changed files with 1 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
tools/transformers/convert2hf.py
View File

@ -142,13 +142,13 @@ if __name__ == "__main__":
parser.add_argument("--max_shard", type=str, default="10GB") # 转换后模型每个切片的大小 parser.add_argument("--max_shard", type=str, default="10GB") # 转换后模型每个切片的大小
args = parser.parse_args() args = parser.parse_args()
dtype = getattr(torch, args.dtype) dtype = getattr(torch, args.dtype)
print_args(args)
def load(fp): def load(fp):
with open(fp, "rb") as f: with open(fp, "rb") as f:
pt_data = torch.load(f, map_location="cpu") pt_data = torch.load(f, map_location="cpu")
return pt_data return pt_data
print_args(args)
folder = args.src_folder folder = args.src_folder
target_folder = args.tgt_folder target_folder = args.tgt_folder