mirror of https://github.com/InternLM/InternLM
fix model_dir
parent
c6485cc612
commit
0809af614d
|
@ -188,8 +188,8 @@ model = AutoModelForCausalLM.from_pretrained("internlm/internlm3-8b-instruct", t
|
||||||
# (Optional) If on low resource devices, you can load model in 4-bit or 8-bit to further save GPU memory via bitsandbytes.
|
# (Optional) If on low resource devices, you can load model in 4-bit or 8-bit to further save GPU memory via bitsandbytes.
|
||||||
# InternLM3 8B in 4bit will cost nearly 8GB GPU memory.
|
# InternLM3 8B in 4bit will cost nearly 8GB GPU memory.
|
||||||
# pip install -U bitsandbytes
|
# pip install -U bitsandbytes
|
||||||
# 8-bit: model = AutoModelForCausalLM.from_pretrained(model_dir, device_map="auto", trust_remote_code=True, load_in_8bit=True)
|
# 8-bit: model = AutoModelForCausalLM.from_pretrained("internlm/internlm3-8b-instruct", device_map="auto", trust_remote_code=True, load_in_8bit=True)
|
||||||
# 4-bit: model = AutoModelForCausalLM.from_pretrained(model_dir, device_map="auto", trust_remote_code=True, load_in_4bit=True)
|
# 4-bit: model = AutoModelForCausalLM.from_pretrained("internlm/internlm3-8b-instruct", device_map="auto", trust_remote_code=True, load_in_4bit=True)
|
||||||
model = model.eval()
|
model = model.eval()
|
||||||
|
|
||||||
messages = [
|
messages = [
|
||||||
|
|
|
@ -186,8 +186,8 @@ model = AutoModelForCausalLM.from_pretrained("internlm/internlm3-8b-instruct", t
|
||||||
# (可选) 如果在低资源设备上,可以通过bitsandbytes加载4-bit或8-bit量化的模型,进一步节省GPU显存.
|
# (可选) 如果在低资源设备上,可以通过bitsandbytes加载4-bit或8-bit量化的模型,进一步节省GPU显存.
|
||||||
# 4-bit 量化的 InternLM3 8B 大约会消耗 8GB 显存.
|
# 4-bit 量化的 InternLM3 8B 大约会消耗 8GB 显存.
|
||||||
# pip install -U bitsandbytes
|
# pip install -U bitsandbytes
|
||||||
# 8-bit: model = AutoModelForCausalLM.from_pretrained(model_dir, device_map="auto", trust_remote_code=True, load_in_8bit=True)
|
# 8-bit: model = AutoModelForCausalLM.from_pretrained("internlm/internlm3-8b-instruct", device_map="auto", trust_remote_code=True, load_in_8bit=True)
|
||||||
# 4-bit: model = AutoModelForCausalLM.from_pretrained(model_dir, device_map="auto", trust_remote_code=True, load_in_4bit=True)
|
# 4-bit: model = AutoModelForCausalLM.from_pretrained("internlm/internlm3-8b-instruct", device_map="auto", trust_remote_code=True, load_in_4bit=True)
|
||||||
model = model.eval()
|
model = model.eval()
|
||||||
|
|
||||||
messages = [
|
messages = [
|
||||||
|
|
|
@ -19,8 +19,8 @@ model = AutoModelForCausalLM.from_pretrained("internlm/internlm3-8b-instruct", t
|
||||||
# (Optional) If on low resource devices, you can load model in 4-bit or 8-bit to further save GPU memory via bitsandbytes.
|
# (Optional) If on low resource devices, you can load model in 4-bit or 8-bit to further save GPU memory via bitsandbytes.
|
||||||
# InternLM3 8B in 4bit will cost nearly 8GB GPU memory.
|
# InternLM3 8B in 4bit will cost nearly 8GB GPU memory.
|
||||||
# pip install -U bitsandbytes
|
# pip install -U bitsandbytes
|
||||||
# 8-bit: model = AutoModelForCausalLM.from_pretrained(model_dir, device_map="auto", trust_remote_code=True, load_in_8bit=True)
|
# 8-bit: model = AutoModelForCausalLM.from_pretrained("internlm/internlm3-8b-instruct", device_map="auto", trust_remote_code=True, load_in_8bit=True)
|
||||||
# 4-bit: model = AutoModelForCausalLM.from_pretrained(model_dir, device_map="auto", trust_remote_code=True, load_in_4bit=True)
|
# 4-bit: model = AutoModelForCausalLM.from_pretrained("internlm/internlm3-8b-instruct", device_map="auto", trust_remote_code=True, load_in_4bit=True)
|
||||||
model = model.eval()
|
model = model.eval()
|
||||||
|
|
||||||
messages = [
|
messages = [
|
||||||
|
@ -42,16 +42,31 @@ response = tokenizer.batch_decode(generated_ids)[0]
|
||||||
To load the InternLM3-8B-Instruct model using ModelScope, use the following code:
|
To load the InternLM3-8B-Instruct model using ModelScope, use the following code:
|
||||||
|
|
||||||
```python
|
```python
|
||||||
from modelscope import snapshot_download, AutoTokenizer, AutoModelForCausalLM
|
|
||||||
import torch
|
import torch
|
||||||
model_dir = snapshot_download('Shanghai_AI_Laboratory/internlm2_5-7b-chat')
|
from modelscope import snapshot_download, AutoTokenizer, AutoModelForCausalLM
|
||||||
tokenizer = AutoTokenizer.from_pretrained(model_dir, device_map="auto", trust_remote_code=True,torch_dtype=torch.float16)
|
model_dir = snapshot_download('Shanghai_AI_Laboratory/internlm3-8b-instruct')
|
||||||
model = AutoModelForCausalLM.from_pretrained(model_dir,device_map="auto", trust_remote_code=True,torch_dtype=torch.float16)
|
tokenizer = AutoTokenizer.from_pretrained(model_dir,trust_remote_code=True)
|
||||||
|
# Set `torch_dtype=torch.float16` to load model in float16, otherwise it will be loaded as float32 and might cause OOM Error.
|
||||||
|
model = AutoModelForCausalLM.from_pretrained(model_dir, trust_remote_code=True, torch_dtype=torch.float16)
|
||||||
|
# (Optional) If on low resource devices, you can load model in 4-bit or 8-bit to further save GPU memory via bitsandbytes.
|
||||||
|
# InternLM3 8B in 4bit will cost nearly 8GB GPU memory.
|
||||||
|
# pip install -U bitsandbytes
|
||||||
|
# 8-bit: model = AutoModelForCausalLM.from_pretrained(model_dir, device_map="auto", trust_remote_code=True, load_in_8bit=True)
|
||||||
|
# 4-bit: model = AutoModelForCausalLM.from_pretrained(model_dir, device_map="auto", trust_remote_code=True, load_in_4bit=True)
|
||||||
model = model.eval()
|
model = model.eval()
|
||||||
response, history = model.chat(tokenizer, "hello", history=[])
|
|
||||||
print(response)
|
messages = [
|
||||||
response, history = model.chat(tokenizer, "please provide three suggestions about time management", history=history)
|
{"role": "system", "content": "You are an AI assistant whose name is InternLM."},
|
||||||
print(response)
|
{"role": "user", "content": "Please tell me five scenic spots in Shanghai"},
|
||||||
|
]
|
||||||
|
tokenized_chat = tokenizer.apply_chat_template(messages, tokenize=True, add_generation_prompt=True, return_tensors="pt")
|
||||||
|
|
||||||
|
generated_ids = model.generate(tokenized_chat, max_new_tokens=512)
|
||||||
|
|
||||||
|
generated_ids = [
|
||||||
|
output_ids[len(input_ids):] for input_ids, output_ids in zip(tokenized_chat, generated_ids)
|
||||||
|
]
|
||||||
|
response = tokenizer.batch_decode(generated_ids)[0]
|
||||||
```
|
```
|
||||||
|
|
||||||
## Dialogue
|
## Dialogue
|
||||||
|
|
|
@ -42,16 +42,29 @@ response = tokenizer.batch_decode(generated_ids)[0]
|
||||||
通过以下的代码从 ModelScope 加载 InternLM2.5-Chat 模型 (可修改模型名称替换不同的模型)
|
通过以下的代码从 ModelScope 加载 InternLM2.5-Chat 模型 (可修改模型名称替换不同的模型)
|
||||||
|
|
||||||
```python
|
```python
|
||||||
from modelscope import snapshot_download, AutoTokenizer, AutoModelForCausalLM
|
|
||||||
import torch
|
import torch
|
||||||
model_dir = snapshot_download('Shanghai_AI_Laboratory/internlm2_5-7b-chat', revision='v1.0.0')
|
from modelscope import snapshot_download, AutoTokenizer, AutoModelForCausalLM
|
||||||
tokenizer = AutoTokenizer.from_pretrained(model_dir, device_map="auto", trust_remote_code=True,torch_dtype=torch.float16)
|
model_dir = snapshot_download('Shanghai_AI_Laboratory/internlm3-8b-instruct')
|
||||||
model = AutoModelForCausalLM.from_pretrained(model_dir,device_map="auto", trust_remote_code=True,torch_dtype=torch.float16)
|
tokenizer = AutoTokenizer.from_pretrained(model_dir,trust_remote_code=True)
|
||||||
model = model.eval()
|
# 设置`torch_dtype=torch.float16`来将模型精度指定为torch.float16,否则可能会因为您的硬件原因造成显存不足的问题。
|
||||||
response, history = model.chat(tokenizer, "hello", history=[])
|
model = AutoModelForCausalLM.from_pretrained(model_dir, device_map="auto", trust_remote_code=True, torch_dtype=torch.float16)
|
||||||
print(response)
|
# (可选) 如果在低资源设备上,可以通过bitsandbytes加载4-bit或8-bit量化的模型,进一步节省GPU显存.
|
||||||
response, history = model.chat(tokenizer, "please provide three suggestions about time management", history=history)
|
# 4-bit 量化的 InternLM3 8B 大约会消耗 8GB 显存.
|
||||||
print(response)
|
# pip install -U bitsandbytes
|
||||||
|
# 8-bit: model = AutoModelForCausalLM.from_pretrained(model_dir, device_map="auto", trust_remote_code=True, load_in_8bit=True)
|
||||||
|
# 4-bit: model = AutoModelForCausalLM.from_pretrained(model_dir, device_map="auto", trust_remote_code=True, load_in_4bit=True)
|
||||||
|
messages = [
|
||||||
|
{"role": "system", "content": "You are an AI assistant whose name is InternLM."},
|
||||||
|
{"role": "user", "content": "Please tell me five scenic spots in Shanghai"},
|
||||||
|
]
|
||||||
|
tokenized_chat = tokenizer.apply_chat_template(messages, tokenize=True, add_generation_prompt=True, return_tensors="pt")
|
||||||
|
|
||||||
|
generated_ids = model.generate(tokenized_chat, max_new_tokens=512)
|
||||||
|
|
||||||
|
generated_ids = [
|
||||||
|
output_ids[len(input_ids):] for input_ids, output_ids in zip(tokenized_chat, generated_ids)
|
||||||
|
]
|
||||||
|
response = tokenizer.batch_decode(generated_ids)[0]
|
||||||
```
|
```
|
||||||
|
|
||||||
## 通过前端网页对话
|
## 通过前端网页对话
|
||||||
|
|
Loading…
Reference in New Issue