update README

README_npu.md

@@ -282,25 +282,43 @@ The loss curve compared with GPU is as follows:
 
 ### Inference
 
-Create the inference script `inference_internlm2_5_7b_chat.py`:
+Create the inference script `inference_internlm3_instruct_8b.py`:
 
 ```python
 import torch
 from transformers import AutoTokenizer, AutoModelForCausalLM
-
-# 若模型已下载，可替换成模型本地路径
-tokenizer = AutoTokenizer.from_pretrained("internlm/internlm2_5-7b-chat", trust_remote_code=True)
-# `torch_dtype=torch.float16`可以令模型以float16精度加载，否则transformers会将模型加载为float32，导致显存不足
-model = AutoModelForCausalLM.from_pretrained("internlm/internlm2_5-7b-chat", torch_dtype=torch.float16, trust_remote_code=True).npu()
+model_dir = "internlm/internlm3-8b-instruct"
+tokenizer = AutoTokenizer.from_pretrained(model_dir, trust_remote_code=True)
+# Set `torch_dtype=torch.float16` to load model in float16, otherwise it will be loaded as float32 and might cause OOM Error.
+model = AutoModelForCausalLM.from_pretrained(model_dir, trust_remote_code=True, torch_dtype=torch.float16).npu()
+# (Optional) If on low resource devices, you can load model in 4-bit or 8-bit to further save GPU memory via bitsandbytes.
+  # InternLM3 8B in 4bit will cost nearly 8GB GPU memory.
+  # pip install -U bitsandbytes
+  # 8-bit: model = AutoModelForCausalLM.from_pretrained(model_dir, trust_remote_code=True, load_in_8bit=True).npu()
+  # 4-bit: model = AutoModelForCausalLM.from_pretrained(model_dir, trust_remote_code=True, load_in_4bit=True).npu()
 model = model.eval()
-response, history = model.chat(tokenizer, "你好，请提供三个管理时间的建议。", history=[])
+system_prompt = """You are an AI assistant whose name is InternLM (书生·浦语).
+- InternLM (书生·浦语) is a conversational language model that is developed by Shanghai AI Laboratory (上海人工智能实验室). It is designed to be helpful, honest, and harmless.
+- InternLM (书生·浦语) can understand and communicate fluently in the language chosen by the user such as English and 中文."""
+messages = [
+    {"role": "system", "content": system_prompt},
+    {"role": "user", "content": "Please tell me five scenic spots in Shanghai"},
+ ]
+tokenized_chat = tokenizer.apply_chat_template(messages, tokenize=True, add_generation_prompt=True, return_tensors="pt").npu()
+generated_ids = model.generate(tokenized_chat, max_new_tokens=1024, temperature=1, repetition_penalty=1.005, top_k=40, top_p=0.8)
+generated_ids = [
+    output_ids[len(input_ids):] for input_ids, output_ids in zip(tokenized_chat, generated_ids)
+]
+prompt = tokenizer.batch_decode(tokenized_chat)[0]
+print(prompt)
+response = tokenizer.batch_decode(generated_ids)[0]
 print(response)
 ```
 
 Execute the inference script:
 
 ```shell
-python inference_internlm2_5_7b_chat.py
+python inference_internlm3_instruct_8b.py
 ```
 
 

README_npu_zh-CN.md

@@ -279,25 +279,44 @@ llamafactory-cli train examples/train_full/internlm3_8b_instruct_full_sft.yaml
 
 ### 推理
 
-新建推理脚本`inference_internlm2_5_7b_chat.py`，推理脚本内容为：
+新建推理脚本`inference_internlm3_instruct_8b.py`，推理脚本内容为：
 
 ```python
 import torch
 from transformers import AutoTokenizer, AutoModelForCausalLM
 
-# 若模型已下载，可替换成模型本地路径
-tokenizer = AutoTokenizer.from_pretrained("internlm/internlm2_5-7b-chat", trust_remote_code=True)
+model_dir = "internlm/internlm3-8b-instruct"
+tokenizer = AutoTokenizer.from_pretrained(model_dir, trust_remote_code=True)
 # `torch_dtype=torch.float16`可以令模型以float16精度加载，否则transformers会将模型加载为float32，导致显存不足
-model = AutoModelForCausalLM.from_pretrained("internlm/internlm2_5-7b-chat", torch_dtype=torch.float16, trust_remote_code=True).npu()
+model = AutoModelForCausalLM.from_pretrained(model_dir, trust_remote_code=True, torch_dtype=torch.float16).npu()
+# （可选）如果在低资源设备上，可以通过bitsandbytes以4位或8位加载模型，从而进一步节省GPU内存。
+  # InternLM3 8B以4位加载将几乎占用8GB的GPU内存.
+  # pip install -U bitsandbytes
+  # 8-bit: model = AutoModelForCausalLM.from_pretrained(model_dir, trust_remote_code=True, load_in_8bit=True).npu()
+  # 4-bit: model = AutoModelForCausalLM.from_pretrained(model_dir, trust_remote_code=True, load_in_4bit=True).npu()
 model = model.eval()
-response, history = model.chat(tokenizer, "你好，请提供三个管理时间的建议。", history=[])
+system_prompt = """You are an AI assistant whose name is InternLM (书生·浦语).
+- InternLM (书生·浦语) is a conversational language model that is developed by Shanghai AI Laboratory (上海人工智能实验室). It is designed to be helpful, honest, and harmless.
+- InternLM (书生·浦语) can understand and communicate fluently in the language chosen by the user such as English and 中文."""
+messages = [
+    {"role": "system", "content": system_prompt},
+    {"role": "user", "content": "Please tell me five scenic spots in Shanghai"},
+ ]
+tokenized_chat = tokenizer.apply_chat_template(messages, tokenize=True, add_generation_prompt=True, return_tensors="pt").npu
+generated_ids = model.generate(tokenized_chat, max_new_tokens=1024, temperature=1, repetition_penalty=1.005, top_k=40, top_p=0.8)
+generated_ids = [
+    output_ids[len(input_ids):] for input_ids, output_ids in zip(tokenized_chat, generated_ids)
+]
+prompt = tokenizer.batch_decode(tokenized_chat)[0]
+print(prompt)
+response = tokenizer.batch_decode(generated_ids)[0]
 print(response)
 ```
 
 执行推理脚本：
 
 ```shell
-python inference_internlm2_5_7b_chat.py
+python inference_internlm3_instruct_8b.py
 ```
 
 ## 开源许可证