From a8a2285561672d1af1a72d0176b5eea399f414e7 Mon Sep 17 00:00:00 2001
From: LYMDLUT <70597027+LYMDLUT@users.noreply.github.com>
Date: Thu, 18 Jan 2024 17:56:31 +0800
Subject: [PATCH] Update README.md

---
 README.md | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/README.md b/README.md
index 64e3769..8a2606c 100644
--- a/README.md
+++ b/README.md
@@ -152,6 +152,10 @@ model_dir = snapshot_download('Shanghai_AI_Laboratory/internlm2-chat-7b')
 tokenizer = AutoTokenizer.from_pretrained(model_dir, device_map="auto", trust_remote_code=True)
 # Set `torch_dtype=torch.float16` to load model in float16, otherwise it will be loaded as float32 and might cause OOM Error.
 model = AutoModelForCausalLM.from_pretrained(model_dir, device_map="auto", trust_remote_code=True, torch_dtype=torch.float16)
+# (Optional) If on low resource devices, you can load model in 4bits or 8 bits to further save GPU memory. InternLM 7B in 4bit will cost nearly 8GB GPU memory. 
+  # pip install -U bitsandbytes
+  # 8bit: model = AutoModelForCausalLM.from_pretrained(model_dir, device_map="auto", trust_remote_code=True, load_in_8bit=True)
+  # 4bit: model = AutoModelForCausalLM.from_pretrained(model_dir, device_map="auto", trust_remote_code=True, load_in_4bit=True)
 model = model.eval()
 response, history = model.chat(tokenizer, "hello", history=[])
 print(response)