github
/
InternLM
mirror of https://github.com/InternLM/InternLM
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

fix model_dir

pull/817/head
pppppM 2025-01-14 12:02:44 +00:00
parent c6485cc612
commit 0809af614d
4 changed files with 51 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
README.md
View File

@ -188,8 +188,8 @@ model = AutoModelForCausalLM.from_pretrained("internlm/internlm3-8b-instruct", t
# (Optional) If on low resource devices, you can load model in 4-bit or 8-bit to further save GPU memory via bitsandbytes. # (Optional) If on low resource devices, you can load model in 4-bit or 8-bit to further save GPU memory via bitsandbytes.
# InternLM3 8B in 4bit will cost nearly 8GB GPU memory. # InternLM3 8B in 4bit will cost nearly 8GB GPU memory.
# pip install -U bitsandbytes # pip install -U bitsandbytes
# 8-bit: model = AutoModelForCausalLM.from_pretrained(model_dir, device_map="auto", trust_remote_code=True, load_in_8bit=True) # 8-bit: model = AutoModelForCausalLM.from_pretrained("internlm/internlm3-8b-instruct", device_map="auto", trust_remote_code=True, load_in_8bit=True)
# 4-bit: model = AutoModelForCausalLM.from_pretrained(model_dir, device_map="auto", trust_remote_code=True, load_in_4bit=True) # 4-bit: model = AutoModelForCausalLM.from_pretrained("internlm/internlm3-8b-instruct", device_map="auto", trust_remote_code=True, load_in_4bit=True)
model = model.eval() model = model.eval()
messages = [ messages = [

4
README_zh-CN.md
View File

@ -186,8 +186,8 @@ model = AutoModelForCausalLM.from_pretrained("internlm/internlm3-8b-instruct", t
# (可选) 如果在低资源设备上，可以通过bitsandbytes加载4-bit或8-bit量化的模型，进一步节省GPU显存. # (可选) 如果在低资源设备上，可以通过bitsandbytes加载4-bit或8-bit量化的模型，进一步节省GPU显存.
# 4-bit 量化的 InternLM3 8B 大约会消耗 8GB 显存. # 4-bit 量化的 InternLM3 8B 大约会消耗 8GB 显存.
# pip install -U bitsandbytes # pip install -U bitsandbytes
# 8-bit: model = AutoModelForCausalLM.from_pretrained(model_dir, device_map="auto", trust_remote_code=True, load_in_8bit=True) # 8-bit: model = AutoModelForCausalLM.from_pretrained("internlm/internlm3-8b-instruct", device_map="auto", trust_remote_code=True, load_in_8bit=True)
# 4-bit: model = AutoModelForCausalLM.from_pretrained(model_dir, device_map="auto", trust_remote_code=True, load_in_4bit=True) # 4-bit: model = AutoModelForCausalLM.from_pretrained("internlm/internlm3-8b-instruct", device_map="auto", trust_remote_code=True, load_in_4bit=True)
model = model.eval() model = model.eval()
messages = [ messages = [

35
chat/README.md
View File

@ -19,8 +19,8 @@ model = AutoModelForCausalLM.from_pretrained("internlm/internlm3-8b-instruct", t
# (Optional) If on low resource devices, you can load model in 4-bit or 8-bit to further save GPU memory via bitsandbytes. # (Optional) If on low resource devices, you can load model in 4-bit or 8-bit to further save GPU memory via bitsandbytes.
# InternLM3 8B in 4bit will cost nearly 8GB GPU memory. # InternLM3 8B in 4bit will cost nearly 8GB GPU memory.
# pip install -U bitsandbytes # pip install -U bitsandbytes
# 8-bit: model = AutoModelForCausalLM.from_pretrained(model_dir, device_map="auto", trust_remote_code=True, load_in_8bit=True) # 8-bit: model = AutoModelForCausalLM.from_pretrained("internlm/internlm3-8b-instruct", device_map="auto", trust_remote_code=True, load_in_8bit=True)
# 4-bit: model = AutoModelForCausalLM.from_pretrained(model_dir, device_map="auto", trust_remote_code=True, load_in_4bit=True) # 4-bit: model = AutoModelForCausalLM.from_pretrained("internlm/internlm3-8b-instruct", device_map="auto", trust_remote_code=True, load_in_4bit=True)
model = model.eval() model = model.eval()
messages = [ messages = [
@ -42,16 +42,31 @@ response = tokenizer.batch_decode(generated_ids)[0]
To load the InternLM3-8B-Instruct model using ModelScope, use the following code: To load the InternLM3-8B-Instruct model using ModelScope, use the following code:
```python ```python
from modelscope import snapshot_download, AutoTokenizer, AutoModelForCausalLM
import torch import torch
model_dir = snapshot_download('Shanghai_AI_Laboratory/internlm2_5-7b-chat') from modelscope import snapshot_download, AutoTokenizer, AutoModelForCausalLM
tokenizer = AutoTokenizer.from_pretrained(model_dir, device_map="auto", trust_remote_code=True,torch_dtype=torch.float16) model_dir = snapshot_download('Shanghai_AI_Laboratory/internlm3-8b-instruct')
model = AutoModelForCausalLM.from_pretrained(model_dir,device_map="auto", trust_remote_code=True,torch_dtype=torch.float16) tokenizer = AutoTokenizer.from_pretrained(model_dir,trust_remote_code=True)
# Set `torch_dtype=torch.float16` to load model in float16, otherwise it will be loaded as float32 and might cause OOM Error.
model = AutoModelForCausalLM.from_pretrained(model_dir, trust_remote_code=True, torch_dtype=torch.float16)
# (Optional) If on low resource devices, you can load model in 4-bit or 8-bit to further save GPU memory via bitsandbytes.
# InternLM3 8B in 4bit will cost nearly 8GB GPU memory.
# pip install -U bitsandbytes
# 8-bit: model = AutoModelForCausalLM.from_pretrained(model_dir, device_map="auto", trust_remote_code=True, load_in_8bit=True)
# 4-bit: model = AutoModelForCausalLM.from_pretrained(model_dir, device_map="auto", trust_remote_code=True, load_in_4bit=True)
model = model.eval() model = model.eval()
response, history = model.chat(tokenizer, "hello", history=[])
print(response) messages = [
response, history = model.chat(tokenizer, "please provide three suggestions about time management", history=history) {"role": "system", "content": "You are an AI assistant whose name is InternLM."},
print(response) {"role": "user", "content": "Please tell me five scenic spots in Shanghai"},
]
tokenized_chat = tokenizer.apply_chat_template(messages, tokenize=True, add_generation_prompt=True, return_tensors="pt")
generated_ids = model.generate(tokenized_chat, max_new_tokens=512)
generated_ids = [
output_ids[len(input_ids):] for input_ids, output_ids in zip(tokenized_chat, generated_ids)
]
response = tokenizer.batch_decode(generated_ids)[0]
``` ```
## Dialogue ## Dialogue

31
chat/README_zh-CN.md
View File

@ -42,16 +42,29 @@ response = tokenizer.batch_decode(generated_ids)[0]
通过以下的代码从 ModelScope 加载 InternLM2.5-Chat 模型 （可修改模型名称替换不同的模型） 通过以下的代码从 ModelScope 加载 InternLM2.5-Chat 模型 （可修改模型名称替换不同的模型）
```python ```python
from modelscope import snapshot_download, AutoTokenizer, AutoModelForCausalLM
import torch import torch
model_dir = snapshot_download('Shanghai_AI_Laboratory/internlm2_5-7b-chat', revision='v1.0.0') from modelscope import snapshot_download, AutoTokenizer, AutoModelForCausalLM
tokenizer = AutoTokenizer.from_pretrained(model_dir, device_map="auto", trust_remote_code=True,torch_dtype=torch.float16) model_dir = snapshot_download('Shanghai_AI_Laboratory/internlm3-8b-instruct')
model = AutoModelForCausalLM.from_pretrained(model_dir,device_map="auto", trust_remote_code=True,torch_dtype=torch.float16) tokenizer = AutoTokenizer.from_pretrained(model_dir,trust_remote_code=True)
model = model.eval() # 设置`torch_dtype=torch.float16`来将模型精度指定为torch.float16，否则可能会因为您的硬件原因造成显存不足的问题。
response, history = model.chat(tokenizer, "hello", history=[]) model = AutoModelForCausalLM.from_pretrained(model_dir, device_map="auto", trust_remote_code=True, torch_dtype=torch.float16)
print(response) # (可选) 如果在低资源设备上，可以通过bitsandbytes加载4-bit或8-bit量化的模型，进一步节省GPU显存.
response, history = model.chat(tokenizer, "please provide three suggestions about time management", history=history) # 4-bit 量化的 InternLM3 8B 大约会消耗 8GB 显存.
print(response) # pip install -U bitsandbytes
# 8-bit: model = AutoModelForCausalLM.from_pretrained(model_dir, device_map="auto", trust_remote_code=True, load_in_8bit=True)
# 4-bit: model = AutoModelForCausalLM.from_pretrained(model_dir, device_map="auto", trust_remote_code=True, load_in_4bit=True)
messages = [
{"role": "system", "content": "You are an AI assistant whose name is InternLM."},
{"role": "user", "content": "Please tell me five scenic spots in Shanghai"},
]
tokenized_chat = tokenizer.apply_chat_template(messages, tokenize=True, add_generation_prompt=True, return_tensors="pt")
generated_ids = model.generate(tokenized_chat, max_new_tokens=512)
generated_ids = [
output_ids[len(input_ids):] for input_ids, output_ids in zip(tokenized_chat, generated_ids)
]
response = tokenizer.batch_decode(generated_ids)[0]
``` ```
## 通过前端网页对话 ## 通过前端网页对话