From acbc2e178a1aae2fcd44030e93c0de5733f5a1c8 Mon Sep 17 00:00:00 2001 From: ZhangErling <45256786+ZhangErling@users.noreply.github.com> Date: Fri, 24 Mar 2023 15:33:42 +0800 Subject: [PATCH 01/10] =?UTF-8?q?=E4=BF=AE=E5=A4=8Dapi=E6=9C=8D=E5=8A=A1?= =?UTF-8?q?=E5=90=AF=E5=8A=A8=E9=94=99=E8=AF=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 第31行【API】->【api】会因为大小写原因找不到API 第34行的下划线【chatglm_6b】->【chatglm-6b】会导致模型加载错误 --- api.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api.py b/api.py index 14a2d57..10f70b6 100644 --- a/api.py +++ b/api.py @@ -28,8 +28,8 @@ async def create_item(request: Request): if __name__ == '__main__': - uvicorn.run('API:app', host='0.0.0.0', port=8000, workers=1) + uvicorn.run('api:app', host='0.0.0.0', port=8000, workers=1) tokenizer = AutoTokenizer.from_pretrained("THUDM/chatglm-6b", trust_remote_code=True) -model = AutoModel.from_pretrained("THUDM/chatglm_6b", trust_remote_code=True).half().cuda() +model = AutoModel.from_pretrained("THUDM/chatglm-6b", trust_remote_code=True).half().cuda() model.eval() From bf39dac0670d4cbb734aaa4664001f1863cecf14 Mon Sep 17 00:00:00 2001 From: holk-h Date: Fri, 24 Mar 2023 18:34:09 +0800 Subject: [PATCH 02/10] Support stream out interruption by using Ctrl+C --- cli_demo.py | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/cli_demo.py b/cli_demo.py index 8a043fb..fea47fc 100644 --- a/cli_demo.py +++ b/cli_demo.py @@ -1,14 +1,15 @@ import os import platform +import signal from transformers import AutoTokenizer, AutoModel -tokenizer = AutoTokenizer.from_pretrained("THUDM/chatglm-6b", trust_remote_code=True) -model = AutoModel.from_pretrained("THUDM/chatglm-6b", trust_remote_code=True).half().cuda() +tokenizer = AutoTokenizer.from_pretrained("./model", trust_remote_code=True) +model = AutoModel.from_pretrained("./model", trust_remote_code=True).half().cuda() model = model.eval() os_name = platform.system() clear_command = 'cls' if os_name == 'Windows' else 'clear' - +stop_stream = False def build_prompt(history): prompt = "欢迎使用 ChatGLM-6B 模型，输入内容即可进行对话，clear 清空对话历史，stop 终止程序" @@ -17,9 +18,13 @@ def build_prompt(history): prompt += f"\n\nChatGLM-6B：{response}" return prompt +def signal_handler(signal, frame): + global stop_stream + stop_stream = True def main(): history = [] + global stop_stream print("欢迎使用 ChatGLM-6B 模型，输入内容即可进行对话，clear 清空对话历史，stop 终止程序") while True: query = input("\n用户：") @@ -32,10 +37,15 @@ def main(): continue count = 0 for response, history in model.stream_chat(tokenizer, query, history=history): - count += 1 - if count % 8 == 0: - os.system(clear_command) - print(build_prompt(history), flush=True) + if stop_stream: + stop_stream = False + break + else: + count += 1 + if count % 8 == 0: + os.system(clear_command) + print(build_prompt(history), flush=True) + signal.signal(signal.SIGINT,signal_handler) os.system(clear_command) print(build_prompt(history), flush=True) From 023c46a317537c9f2a6bdfc4916c77b24a00d868 Mon Sep 17 00:00:00 2001 From: littlepanda0716 Date: Sat, 25 Mar 2023 18:59:11 +0800 Subject: [PATCH 03/10] update api.py --- api.py | 28 +++++++++++++++++++++++++--- 1 file changed, 25 insertions(+), 3 deletions(-) diff --git a/api.py b/api.py index 10f70b6..4ad1db6 100644 --- a/api.py +++ b/api.py @@ -1,6 +1,19 @@ from fastapi import FastAPI, Request from transformers import AutoTokenizer, AutoModel import uvicorn, json, datetime +import torch + +DEVICE = "cuda" +DEVICE_ID = "0" +CUDA_DEVICE = f"{DEVICE}:{DEVICE_ID}" if DEVICE_ID else DEVICE + + +def torch_gc(): + if torch.cuda.is_available(): + with torch.cuda.device(CUDA_DEVICE): + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + app = FastAPI() @@ -13,7 +26,15 @@ async def create_item(request: Request): json_post_list = json.loads(json_post) prompt = json_post_list.get('prompt') history = json_post_list.get('history') - response, history = model.chat(tokenizer, prompt, history=history) + max_length = json_post_list.get('max_length') + top_p = json_post_list.get('top_p') + temperature = json_post_list.get('temperature') + response, history = model.chat(tokenizer, + prompt, + history=history, + max_length=max_length if max_length else 2048, + top_p=top_p if top_p else 0.7, + temperature=temperature if temperature else 0.95) now = datetime.datetime.now() time = now.strftime("%Y-%m-%d %H:%M:%S") answer = { @@ -24,12 +45,13 @@ async def create_item(request: Request): } log = "[" + time + "] " + '", prompt:"' + prompt + '", response:"' + repr(response) + '"' print(log) + torch_gc() return answer if __name__ == '__main__': - uvicorn.run('api:app', host='0.0.0.0', port=8000, workers=1) + uvicorn.run(app, host='0.0.0.0', port=8000, workers=1) tokenizer = AutoTokenizer.from_pretrained("THUDM/chatglm-6b", trust_remote_code=True) -model = AutoModel.from_pretrained("THUDM/chatglm-6b", trust_remote_code=True).half().cuda() +model = AutoModel.from_pretrained("THUDM/chatglm_6b", trust_remote_code=True).half().cuda() model.eval() From 9addb875ca503a37804fb82d55991b2f7392f8f9 Mon Sep 17 00:00:00 2001 From: duzx16 Date: Tue, 28 Mar 2023 16:58:38 +0800 Subject: [PATCH 04/10] Add links --- README.md | 6 ++++++ README_en.md | 7 +++++++ 2 files changed, 13 insertions(+) diff --git a/README.md b/README.md index 58f3751..075ce78 100644 --- a/README.md +++ b/README.md @@ -14,6 +14,12 @@ ChatGLM-6B 使用了和 ChatGPT 相似的技术，针对中文问答和对话进 **[2023/03/19]** 增加流式输出接口 `stream_chat`，已更新到网页版和命令行 Demo。修复输出中的中文标点。增加量化后的模型 [ChatGLM-6B-INT4](https://huggingface.co/THUDM/chatglm-6b-int4) +## 友情链接 +以下是部分基于本仓库开发的开源项目： +* [ChatGLM-MNN](https://github.com/wangzhaode/ChatGLM-MNN): 一个基于 MNN 的 ChatGLM-6B C++ 推理实现，支持根据显存大小动态分配计算任务给 GPU 和 CPU +* [ChatGLM-Tuning](https://github.com/mymusise/ChatGLM-Tuning): 基于 LoRA 对 ChatGLM-6B 进行微调 + +如果你有其他好的项目的话，欢迎参照上述格式添加到README中并提出 [PR](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/creating-a-pull-request-from-a-fork). ## 使用方式 diff --git a/README_en.md b/README_en.md index 3a8b17b..584e97f 100644 --- a/README_en.md +++ b/README_en.md @@ -13,6 +13,13 @@ Try the [online demo](https://huggingface.co/spaces/ysharma/ChatGLM-6b_Gradio_St **[2023/03/19]** Add streaming output function `stream_chat`, already applied in web and CLI demo. Fix Chinese punctuations in output. Add quantized model [ChatGLM-6B-INT4](https://huggingface.co/THUDM/chatglm-6b-int4). +## Projects +The following are some open source projects developed based on this repository: +* [ChatGLM-MNN](https://github.com/wangzhaode/ChatGLM-MNN): An [MNN](https://github.com/alibaba/MNN)-based implementation of ChatGLM-6B C++ inference, which supports dynamic allocation of computing tasks to GPU and CPU according to the size of GPU memory +* [ChatGLM-Tuning](https://github.com/mymusise/ChatGLM-Tuning): Fine-tuning ChatGLM-6B based on LoRA + +If you have other good projects, please refer to the above format to add to README and propose [PR](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/creating-a-pull-request-from-a-fork). + ## Getting Started ### Hardware Requirements From 1c6002f3f18496217c7c0d147ef34c291c6da65d Mon Sep 17 00:00:00 2001 From: duzx16 Date: Tue, 28 Mar 2023 19:29:41 +0800 Subject: [PATCH 05/10] Fix typo --- cli_demo.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cli_demo.py b/cli_demo.py index 8a043fb..0c6ed13 100644 --- a/cli_demo.py +++ b/cli_demo.py @@ -2,8 +2,8 @@ import os import platform from transformers import AutoTokenizer, AutoModel -tokenizer = AutoTokenizer.from_pretrained("THUDM/chatglm-6b", trust_remote_code=True) -model = AutoModel.from_pretrained("THUDM/chatglm-6b", trust_remote_code=True).half().cuda() +tokenizer = AutoTokenizer.from_pretrained("/Users/zhengxiaodu/Downloads/chatglm-6b", trust_remote_code=True) +model = AutoModel.from_pretrained("/Users/zhengxiaodu/Downloads/chatglm-6b", trust_remote_code=True).half().to("mps") model = model.eval() os_name = platform.system() From c6790a09f05ba6a23b073021bfa6c3df177442fc Mon Sep 17 00:00:00 2001 From: duzx16 Date: Tue, 28 Mar 2023 19:45:17 +0800 Subject: [PATCH 06/10] Fix typos Move model instantiation --- api.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/api.py b/api.py index 4ad1db6..693c70a 100644 --- a/api.py +++ b/api.py @@ -50,8 +50,7 @@ async def create_item(request: Request): if __name__ == '__main__': + tokenizer = AutoTokenizer.from_pretrained("THUDM/chatglm-6b", trust_remote_code=True) + model = AutoModel.from_pretrained("THUDM/chatglm-6b", trust_remote_code=True).half().cuda() + model.eval() uvicorn.run(app, host='0.0.0.0', port=8000, workers=1) - -tokenizer = AutoTokenizer.from_pretrained("THUDM/chatglm-6b", trust_remote_code=True) -model = AutoModel.from_pretrained("THUDM/chatglm_6b", trust_remote_code=True).half().cuda() -model.eval() From 6fc8141a9c9c57fe2e4897ff93491a9a61b0fb16 Mon Sep 17 00:00:00 2001 From: duzx16 Date: Tue, 28 Mar 2023 19:48:21 +0800 Subject: [PATCH 07/10] Revert "Fix typo" This reverts commit 1c6002f3f18496217c7c0d147ef34c291c6da65d. --- cli_demo.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cli_demo.py b/cli_demo.py index 0c6ed13..8a043fb 100644 --- a/cli_demo.py +++ b/cli_demo.py @@ -2,8 +2,8 @@ import os import platform from transformers import AutoTokenizer, AutoModel -tokenizer = AutoTokenizer.from_pretrained("/Users/zhengxiaodu/Downloads/chatglm-6b", trust_remote_code=True) -model = AutoModel.from_pretrained("/Users/zhengxiaodu/Downloads/chatglm-6b", trust_remote_code=True).half().to("mps") +tokenizer = AutoTokenizer.from_pretrained("THUDM/chatglm-6b", trust_remote_code=True) +model = AutoModel.from_pretrained("THUDM/chatglm-6b", trust_remote_code=True).half().cuda() model = model.eval() os_name = platform.system() From 343e7bc7b6126718f10dc57ef3c911958c4b273b Mon Sep 17 00:00:00 2001 From: duzx16 Date: Tue, 28 Mar 2023 19:52:32 +0800 Subject: [PATCH 08/10] Fix model path --- cli_demo.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/cli_demo.py b/cli_demo.py index fea47fc..1c3ff2b 100644 --- a/cli_demo.py +++ b/cli_demo.py @@ -3,14 +3,15 @@ import platform import signal from transformers import AutoTokenizer, AutoModel -tokenizer = AutoTokenizer.from_pretrained("./model", trust_remote_code=True) -model = AutoModel.from_pretrained("./model", trust_remote_code=True).half().cuda() +tokenizer = AutoTokenizer.from_pretrained("THUDM/chatglm-6b", trust_remote_code=True) +model = AutoModel.from_pretrained("THUDM/chatglm-6b", trust_remote_code=True).half().cuda() model = model.eval() os_name = platform.system() clear_command = 'cls' if os_name == 'Windows' else 'clear' stop_stream = False + def build_prompt(history): prompt = "欢迎使用 ChatGLM-6B 模型，输入内容即可进行对话，clear 清空对话历史，stop 终止程序" for query, response in history: @@ -18,10 +19,12 @@ def build_prompt(history): prompt += f"\n\nChatGLM-6B：{response}" return prompt + def signal_handler(signal, frame): global stop_stream stop_stream = True + def main(): history = [] global stop_stream @@ -45,7 +48,7 @@ def main(): if count % 8 == 0: os.system(clear_command) print(build_prompt(history), flush=True) - signal.signal(signal.SIGINT,signal_handler) + signal.signal(signal.SIGINT, signal_handler) os.system(clear_command) print(build_prompt(history), flush=True) From 7d7d87c4bd26d84485e09d284437fc0682deb45f Mon Sep 17 00:00:00 2001 From: duzx16 Date: Tue, 28 Mar 2023 21:18:59 +0800 Subject: [PATCH 09/10] Update README --- README.md | 2 +- README_en.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 075ce78..2f59ad6 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,7 @@ ChatGLM-6B 使用了和 ChatGPT 相似的技术，针对中文问答和对话进 ## 友情链接 以下是部分基于本仓库开发的开源项目： -* [ChatGLM-MNN](https://github.com/wangzhaode/ChatGLM-MNN): 一个基于 MNN 的 ChatGLM-6B C++ 推理实现，支持根据显存大小动态分配计算任务给 GPU 和 CPU +* [ChatGLM-MNN](https://github.com/wangzhaode/ChatGLM-MNN): 一个基于 MNN 的 ChatGLM-6B C++ 推理实现，支持根据显存大小自动分配计算任务给 GPU 和 CPU * [ChatGLM-Tuning](https://github.com/mymusise/ChatGLM-Tuning): 基于 LoRA 对 ChatGLM-6B 进行微调 如果你有其他好的项目的话，欢迎参照上述格式添加到README中并提出 [PR](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/creating-a-pull-request-from-a-fork). diff --git a/README_en.md b/README_en.md index 584e97f..7b84f9c 100644 --- a/README_en.md +++ b/README_en.md @@ -15,7 +15,7 @@ Try the [online demo](https://huggingface.co/spaces/ysharma/ChatGLM-6b_Gradio_St ## Projects The following are some open source projects developed based on this repository: -* [ChatGLM-MNN](https://github.com/wangzhaode/ChatGLM-MNN): An [MNN](https://github.com/alibaba/MNN)-based implementation of ChatGLM-6B C++ inference, which supports dynamic allocation of computing tasks to GPU and CPU according to the size of GPU memory +* [ChatGLM-MNN](https://github.com/wangzhaode/ChatGLM-MNN): An [MNN](https://github.com/alibaba/MNN)-based implementation of ChatGLM-6B C++ inference, which supports automatic allocation of computing tasks to GPU and CPU according to the size of GPU memory * [ChatGLM-Tuning](https://github.com/mymusise/ChatGLM-Tuning): Fine-tuning ChatGLM-6B based on LoRA If you have other good projects, please refer to the above format to add to README and propose [PR](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/creating-a-pull-request-from-a-fork). From fc55c075fe8a2c403917d2c7b04d3f0ef3b0e0f2 Mon Sep 17 00:00:00 2001 From: duzx16 Date: Tue, 28 Mar 2023 21:35:52 +0800 Subject: [PATCH 10/10] Update README --- README.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 2f59ad6..814277d 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,10 @@ ChatGLM-6B 使用了和 ChatGPT 相似的技术，针对中文问答和对话进 * [ChatGLM-MNN](https://github.com/wangzhaode/ChatGLM-MNN): 一个基于 MNN 的 ChatGLM-6B C++ 推理实现，支持根据显存大小自动分配计算任务给 GPU 和 CPU * [ChatGLM-Tuning](https://github.com/mymusise/ChatGLM-Tuning): 基于 LoRA 对 ChatGLM-6B 进行微调 -如果你有其他好的项目的话，欢迎参照上述格式添加到README中并提出 [PR](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/creating-a-pull-request-from-a-fork). +以下是部分针对本项目的教程/文档： +* [Windows部署文档](https://github.com/ZhangErling/ChatGLM-6B/blob/main/deployment_windows.md) + +如果你有其他好的项目/教程的话，欢迎参照上述格式添加到README中并提出 [PR](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/creating-a-pull-request-from-a-fork). ## 使用方式