From c133b53d51665fb4a228deb37c1f3b16b1ca5e01 Mon Sep 17 00:00:00 2001 From: LemonQu <117093997+LemonQu-GIT@users.noreply.github.com> Date: Sun, 19 Mar 2023 22:03:02 +0800 Subject: [PATCH 01/54] Add API.py --- API.py | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 API.py diff --git a/API.py b/API.py new file mode 100644 index 0000000..7149f78 --- /dev/null +++ b/API.py @@ -0,0 +1,33 @@ +from typing import Optional +from fastapi import FastAPI, Request +from pydantic import BaseModel +from transformers import AutoTokenizer, AutoModel +import uvicorn, json, time, datetime, os, platform + +app = FastAPI() +@app.post("/") +async def create_item(request: Request): + global history, model, tokenizer + jsonPostRaw = await request.json() + jsonPost = json.dumps(jsonPostRaw) + jsonPostList = json.loads(jsonPost) + prompt = jsonPostList.get('prompt') + response, history = model.chat(tokenizer, prompt, history=history) + now = datetime.datetime.now() + time = now.strftime("%Y-%m-%d %H:%M:%S") + answer = { + "response":response, + "status":200, + "time":time + } + log = "["+time+"] "+'device:"'+jsonPostList.get('device')+'", prompt:"'+prompt+'", response:"'+repr(response)+'"' + print(log) + return answer + +if __name__ == '__main__': + uvicorn.run('API:app',host='0.0.0.0',port=8000,workers=1) + +history = [] +tokenizer = AutoTokenizer.from_pretrained("THUDM/chatglm-6b", trust_remote_code=True) +model = AutoModel.from_pretrained("THUDM/chatglm-6b", trust_remote_code=True).half().quantize(4).cuda() +model = model.eval() From cbf3d2b666503b76e0ba0bf26aac531e98ad9588 Mon Sep 17 00:00:00 2001 From: duzx16 Date: Tue, 21 Mar 2023 15:27:17 +0800 Subject: [PATCH 02/54] Fix transformers version --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 8b52fba..2948480 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ protobuf>=3.19.5,<3.20.1 -transformers>=4.26.1 +transformers==4.26.1 icetk cpm_kernels torch>=1.10 From 292b5684c18a1765bf2570739e5cf2dcb03cca41 Mon Sep 17 00:00:00 2001 From: Shaw Date: Tue, 21 Mar 2023 23:32:49 +0800 Subject: [PATCH 03/54] Update link to GLM-130B --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 04fd4fe..734ce70 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ ChatGLM-6B 是一个开源的、支持中英双语的对话语言模型，基于 [General Language Model (GLM)](https://github.com/THUDM/GLM) 架构，具有 62 亿参数。结合模型量化技术，用户可以在消费级的显卡上进行本地部署（INT4 量化级别下最低只需 6GB 显存）。 ChatGLM-6B 使用了和 ChatGPT 相似的技术，针对中文问答和对话进行了优化。经过约 1T 标识符的中英双语训练，辅以监督微调、反馈自助、人类反馈强化学习等技术的加持，62 亿参数的 ChatGLM-6B 已经能生成相当符合人类偏好的回答。更多信息请参考我们的[博客](https://chatglm.cn/blog)。 -不过，由于 ChatGLM-6B 的规模较小，目前已知其具有相当多的[**局限性**](#局限性)，如事实性/数学逻辑错误，可能生成有害/有偏见内容，较弱的上下文能力，自我认知混乱，以及对英文指示生成与中文指示完全矛盾的内容。请大家在使用前了解这些问题，以免产生误解。 +不过，由于 ChatGLM-6B 的规模较小，目前已知其具有相当多的[**局限性**](#局限性)，如事实性/数学逻辑错误，可能生成有害/有偏见内容，较弱的上下文能力，自我认知混乱，以及对英文指示生成与中文指示完全矛盾的内容。请大家在使用前了解这些问题，以免产生误解。更大的基于1300亿参数[GLM-130B](https://github.com/THUDM/GLM-130B)的ChatGLM正在内测开发中。 *Read this in [English](README_en.md).* From ef6b3ff35eab946ce99479ada2154a1fa2318ace Mon Sep 17 00:00:00 2001 From: AdamBear Date: Wed, 22 Mar 2023 15:11:21 +0800 Subject: [PATCH 04/54] support stream chat --- web_demo2.py | 29 +++++++++++++++++------------ 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/web_demo2.py b/web_demo2.py index 6946a15..fafeb5d 100644 --- a/web_demo2.py +++ b/web_demo2.py @@ -11,8 +11,8 @@ st.set_page_config( @st.cache_resource def get_model(): - tokenizer = AutoTokenizer.from_pretrained("/THUDM/chatglm-6b", trust_remote_code=True) - model = AutoModel.from_pretrained("THUDM/chatglm-6b", trust_remote_code=True).half().cuda() + tokenizer = AutoTokenizer.from_pretrained("/data/chatglm-6b", trust_remote_code=True) + model = AutoModel.from_pretrained("/data/chatglm-6b", trust_remote_code=True).half().cuda() model = model.eval() return tokenizer, model @@ -25,26 +25,31 @@ def predict(input, history=None): tokenizer, model = get_model() if history is None: history = [] - response, history = model.chat(tokenizer, input, history) - #updates = [] - for i, (query, response) in enumerate(history): - #updates.append("用户：" + query) - message(query, avatar_style="big-smile", key=str(i) + "_user") - #updates.append("ChatGLM-6B：" + response) - message(response, avatar_style="bottts", key=str(i)) + with container: + if len(history) > 0: + for i, (query, response) in enumerate(history): + message(query, avatar_style="big-smile", key=str(i) + "_user") + message(response, avatar_style="bottts", key=str(i)) - # if len(updates) < MAX_BOXES: - # updates = updates + [""] * (MAX_BOXES - len(updates)) + message(input, avatar_style="big-smile", key=str(len(history)) + "_user") + st.write("AI正在回复:") + with st.empty(): + for response, history in model.stream_chat(tokenizer, input, history): + query, response = history[-1] + st.write(response) return history +container = st.container() + # create a prompt text for the text generation prompt_text = st.text_area(label="用户命令输入", height = 100, placeholder="请在这儿输入您的命令") + if 'state' not in st.session_state: st.session_state['state'] = [] @@ -53,4 +58,4 @@ if st.button("发送", key="predict"): # text generation st.session_state["state"] = predict(prompt_text, st.session_state["state"]) - st.balloons() \ No newline at end of file + st.session_state["state"] \ No newline at end of file From ee76342382e5894f45f44ad04bc6b047df26cce4 Mon Sep 17 00:00:00 2001 From: AdamBear Date: Wed, 22 Mar 2023 15:16:08 +0800 Subject: [PATCH 05/54] merge --- web_demo2.py | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/web_demo2.py b/web_demo2.py index ede0872..ff78a99 100644 --- a/web_demo2.py +++ b/web_demo2.py @@ -11,13 +11,8 @@ st.set_page_config( @st.cache_resource def get_model(): -<<<<<<< HEAD - tokenizer = AutoTokenizer.from_pretrained("/data/chatglm-6b", trust_remote_code=True) - model = AutoModel.from_pretrained("/data/chatglm-6b", trust_remote_code=True).half().cuda() -======= tokenizer = AutoTokenizer.from_pretrained("THUDM/chatglm-6b", trust_remote_code=True) model = AutoModel.from_pretrained("THUDM/chatglm-6b", trust_remote_code=True).half().cuda() ->>>>>>> 292b5684c18a1765bf2570739e5cf2dcb03cca41 model = model.eval() return tokenizer, model @@ -31,7 +26,6 @@ def predict(input, history=None): if history is None: history = [] -<<<<<<< HEAD with container: if len(history) > 0: for i, (query, response) in enumerate(history): @@ -45,12 +39,6 @@ def predict(input, history=None): query, response = history[-1] st.write(response) -======= - for i, (query, response) in enumerate(history): - message(query, avatar_style="big-smile", key=str(i) + "_user") - message(response, avatar_style="bottts", key=str(i)) - ->>>>>>> 292b5684c18a1765bf2570739e5cf2dcb03cca41 return history From 5513dd7d2c61023adb5a48d4b723e8ff35163837 Mon Sep 17 00:00:00 2001 From: duzx16 Date: Thu, 23 Mar 2023 14:46:05 +0800 Subject: [PATCH 06/54] Delete session_state show --- web_demo2.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/web_demo2.py b/web_demo2.py index ff78a99..4e1f0e4 100644 --- a/web_demo2.py +++ b/web_demo2.py @@ -57,5 +57,3 @@ if st.button("发送", key="predict"): with st.spinner("AI正在思考，请稍等........"): # text generation st.session_state["state"] = predict(prompt_text, st.session_state["state"]) - - st.session_state["state"] \ No newline at end of file From 44356fcf3b543016bcc7af4eb83e5991b0295331 Mon Sep 17 00:00:00 2001 From: Yuvraj Sharma <48665385+yvrjsharma@users.noreply.github.com> Date: Thu, 23 Mar 2023 18:09:51 +0530 Subject: [PATCH 07/54] Added a link to streaming demo on Huggingface --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 734ce70..3585aa9 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,8 @@ ChatGLM-6B 使用了和 ChatGPT 相似的技术，针对中文问答和对话进 ## 更新信息 **[2023/03/19]** 增加流式输出接口 `stream_chat`，已更新到网页版和命令行 Demo。修复输出中的中文标点。增加量化后的模型 [ChatGLM-6B-INT4](https://huggingface.co/THUDM/chatglm-6b-int4) +上的流媒体演示 [Huggingface Spaces](https://huggingface.co/spaces/ysharma/ChatGLM-6b_Gradio_Streaming) + ## 使用方式 ### 硬件需求 From af7f1d46709412056367dad52ea8830c9866b87d Mon Sep 17 00:00:00 2001 From: duzx16 Date: Thu, 23 Mar 2023 21:23:29 +0800 Subject: [PATCH 08/54] Update online demo --- README.md | 4 +++- README_en.md | 2 ++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 3585aa9..82aa852 100644 --- a/README.md +++ b/README.md @@ -7,12 +7,14 @@ ChatGLM-6B 使用了和 ChatGPT 相似的技术，针对中文问答和对话进 不过，由于 ChatGLM-6B 的规模较小，目前已知其具有相当多的[**局限性**](#局限性)，如事实性/数学逻辑错误，可能生成有害/有偏见内容，较弱的上下文能力，自我认知混乱，以及对英文指示生成与中文指示完全矛盾的内容。请大家在使用前了解这些问题，以免产生误解。更大的基于1300亿参数[GLM-130B](https://github.com/THUDM/GLM-130B)的ChatGLM正在内测开发中。 +欢迎体验 Huggingface Spaces 上的[在线演示](https://huggingface.co/spaces/ysharma/ChatGLM-6b_Gradio_Streaming)。 + + *Read this in [English](README_en.md).* ## 更新信息 **[2023/03/19]** 增加流式输出接口 `stream_chat`，已更新到网页版和命令行 Demo。修复输出中的中文标点。增加量化后的模型 [ChatGLM-6B-INT4](https://huggingface.co/THUDM/chatglm-6b-int4) -上的流媒体演示 [Huggingface Spaces](https://huggingface.co/spaces/ysharma/ChatGLM-6b_Gradio_Streaming) ## 使用方式 diff --git a/README_en.md b/README_en.md index b4dcfe8..cb5e792 100644 --- a/README_en.md +++ b/README_en.md @@ -6,6 +6,8 @@ ChatGLM-6B is an open bilingual language model based on [General Language Model ChatGLM-6B uses technology similar to ChatGPT, optimized for Chinese QA and dialogue. The model is trained for about 1T tokens of Chinese and English corpus, supplemented by supervised fine-tuning, feedback bootstrap, and reinforcement learning wit human feedback. With only about 6.2 billion parameters, the model is able to generate answers that are in line with human preference. +Try the [online demo](https://huggingface.co/spaces/ysharma/ChatGLM-6b_Gradio_Streaming) on Huggingface Spaces. + ## Update **[2023/03/19]** Add streaming output function `stream_chat`, already applied in web and CLI demo. Fix Chinese punctuations in output. Add quantized model [ChatGLM-6B-INT4](https://huggingface.co/THUDM/chatglm-6b-int4). From b0c2b47f5eac8f3e5d7e5f05f227e894877cd134 Mon Sep 17 00:00:00 2001 From: duzx16 Date: Thu, 23 Mar 2023 21:42:43 +0800 Subject: [PATCH 09/54] Add history in API --- API.py | 36 +++++++++++++++++++----------------- 1 file changed, 19 insertions(+), 17 deletions(-) diff --git a/API.py b/API.py index 7149f78..14a2d57 100644 --- a/API.py +++ b/API.py @@ -1,33 +1,35 @@ -from typing import Optional from fastapi import FastAPI, Request -from pydantic import BaseModel from transformers import AutoTokenizer, AutoModel -import uvicorn, json, time, datetime, os, platform +import uvicorn, json, datetime app = FastAPI() + + @app.post("/") async def create_item(request: Request): - global history, model, tokenizer - jsonPostRaw = await request.json() - jsonPost = json.dumps(jsonPostRaw) - jsonPostList = json.loads(jsonPost) - prompt = jsonPostList.get('prompt') + global model, tokenizer + json_post_raw = await request.json() + json_post = json.dumps(json_post_raw) + json_post_list = json.loads(json_post) + prompt = json_post_list.get('prompt') + history = json_post_list.get('history') response, history = model.chat(tokenizer, prompt, history=history) now = datetime.datetime.now() time = now.strftime("%Y-%m-%d %H:%M:%S") answer = { - "response":response, - "status":200, - "time":time + "response": response, + "history": history, + "status": 200, + "time": time } - log = "["+time+"] "+'device:"'+jsonPostList.get('device')+'", prompt:"'+prompt+'", response:"'+repr(response)+'"' + log = "[" + time + "] " + '", prompt:"' + prompt + '", response:"' + repr(response) + '"' print(log) return answer -if __name__ == '__main__': - uvicorn.run('API:app',host='0.0.0.0',port=8000,workers=1) -history = [] +if __name__ == '__main__': + uvicorn.run('API:app', host='0.0.0.0', port=8000, workers=1) + tokenizer = AutoTokenizer.from_pretrained("THUDM/chatglm-6b", trust_remote_code=True) -model = AutoModel.from_pretrained("THUDM/chatglm-6b", trust_remote_code=True).half().quantize(4).cuda() -model = model.eval() +model = AutoModel.from_pretrained("THUDM/chatglm_6b", trust_remote_code=True).half().cuda() +model.eval() From 955d4750798fad3121aa4470dc20d18339b810c6 Mon Sep 17 00:00:00 2001 From: duzx16 Date: Thu, 23 Mar 2023 21:59:54 +0800 Subject: [PATCH 10/54] Add API deployment --- README.md | 25 ++++++++++++++++++++++++- README_en.md | 23 +++++++++++++++++++++++ API.py => api.py | 0 3 files changed, 47 insertions(+), 1 deletion(-) rename API.py => api.py (100%) diff --git a/README.md b/README.md index 82aa852..84f7bc1 100644 --- a/README.md +++ b/README.md @@ -13,6 +13,8 @@ ChatGLM-6B 使用了和 ChatGPT 相似的技术，针对中文问答和对话进 *Read this in [English](README_en.md).* ## 更新信息 +**[2023/03/23]** 增加API部署，感谢 [@LemonQu-GIT](https://github.com/LemonQu-GIT) + **[2023/03/19]** 增加流式输出接口 `stream_chat`，已更新到网页版和命令行 Demo。修复输出中的中文标点。增加量化后的模型 [ChatGLM-6B-INT4](https://huggingface.co/THUDM/chatglm-6b-int4) @@ -78,7 +80,7 @@ python web_demo.py 程序会运行一个 Web Server，并输出地址。在浏览器中打开输出的地址即可使用。最新版 Demo 实现了打字机效果，速度体验大大提升。注意，由于国内 Gradio 的网络访问较为缓慢，启用 `demo.queue().launch(share=True, inbrowser=True)` 时所有网络会经过 Gradio 服务器转发，导致打字机体验大幅下降，现在默认启动方式已经改为 `share=False`，如有需要公网访问的需求，可以重新修改为 `share=True` 启动。 -感谢[@AdamBear](https://github.com/AdamBear) 实现了基于 Streamlit 的网页版 Demo，运行方式见[#117](https://github.com/THUDM/ChatGLM-6B/pull/117). +感谢 [@AdamBear](https://github.com/AdamBear) 实现了基于 Streamlit 的网页版 Demo，运行方式见[#117](https://github.com/THUDM/ChatGLM-6B/pull/117). #### 命令行 Demo @@ -92,6 +94,27 @@ python cli_demo.py 程序会在命令行中进行交互式的对话，在命令行中输入指示并回车即可生成回复，输入`clear`可以清空对话历史，输入`stop`终止程序。 +## API部署 +首先需要安装额外的依赖`pip install fastapi uvicorn`，然后运行仓库中的[api.py](api.py)： +```shell +python api.py +``` +默认部署在本地的8000端口，通过POST方法进行调用 +```shell +curl -X POST "http://127.0.0.1:8000" \ + -H 'Content-Type: application/json' \ + -d '{"prompt": "你好", "history": []}' +``` +得到的返回值为 +```shell +{ + "response":"你好👋！我是人工智能助手 ChatGLM-6B，很高兴见到你，欢迎问我任何问题。", + "history":[["你好","你好👋！我是人工智能助手 ChatGLM-6B，很高兴见到你，欢迎问我任何问题。"]], + "status":200, + "time":"2023-03-23 21:38:40" +} +``` + ## 低成本部署 ### 模型量化 默认情况下，模型以 FP16 精度加载，运行上述代码需要大概 13GB 显存。如果你的 GPU 显存有限，可以尝试以量化方式加载模型，使用方法如下： diff --git a/README_en.md b/README_en.md index cb5e792..ba9366d 100644 --- a/README_en.md +++ b/README_en.md @@ -9,6 +9,8 @@ ChatGLM-6B uses technology similar to ChatGPT, optimized for Chinese QA and dial Try the [online demo](https://huggingface.co/spaces/ysharma/ChatGLM-6b_Gradio_Streaming) on Huggingface Spaces. ## Update +**[2023/03/23]** Add API deployment, thanks to [@LemonQu-GIT](https://github.com/LemonQu-GIT) + **[2023/03/19]** Add streaming output function `stream_chat`, already applied in web and CLI demo. Fix Chinese punctuations in output. Add quantized model [ChatGLM-6B-INT4](https://huggingface.co/THUDM/chatglm-6b-int4). ## Getting Started @@ -86,6 +88,27 @@ python cli_demo.py The command runs an interactive program in the shell. Type your instruction in the shell and hit enter to generate the response. Type `clear` to clear the dialogue history and `stop` to terminate the program. +## API Deployment +First install the additional dependency `pip install fastapi uvicorn`. The run [api.py](api.py) in the repo. +```shell +python api.py +``` +By default the api runs at the`8000`port of the local machine. You can call the API via +```shell +curl -X POST "http://127.0.0.1:8000" \ + -H 'Content-Type: application/json' \ + -d '{"prompt": "你好", "history": []}' +``` +The returned value is +```shell +{ + "response":"你好👋！我是人工智能助手 ChatGLM-6B，很高兴见到你，欢迎问我任何问题。", + "history":[["你好","你好👋！我是人工智能助手 ChatGLM-6B，很高兴见到你，欢迎问我任何问题。"]], + "status":200, + "time":"2023-03-23 21:38:40" +} +``` + ## Deployment ### Quantization diff --git a/API.py b/api.py similarity index 100% rename from API.py rename to api.py From 6b13f660bc6fe8f8caee572456ec105d816e2561 Mon Sep 17 00:00:00 2001 From: duzx16 Date: Thu, 23 Mar 2023 22:06:15 +0800 Subject: [PATCH 11/54] Add chatglm-6b-int4-qe --- README.md | 9 ++++++++- README_en.md | 7 ++++++- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 84f7bc1..14680b3 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ ChatGLM-6B 使用了和 ChatGPT 相似的技术，针对中文问答和对话进 *Read this in [English](README_en.md).* ## 更新信息 -**[2023/03/23]** 增加API部署，感谢 [@LemonQu-GIT](https://github.com/LemonQu-GIT) +**[2023/03/23]** 增加API部署（感谢 [@LemonQu-GIT](https://github.com/LemonQu-GIT)），增加Embedding量化模型[ChatGLM-6B-INT4-QE](https://huggingface.co/THUDM/chatglm-6b-int4-qe) **[2023/03/19]** 增加流式输出接口 `stream_chat`，已更新到网页版和命令行 Demo。修复输出中的中文标点。增加量化后的模型 [ChatGLM-6B-INT4](https://huggingface.co/THUDM/chatglm-6b-int4) @@ -133,6 +133,13 @@ model = AutoModel.from_pretrained("THUDM/chatglm-6b", trust_remote_code=True).ha model = AutoModel.from_pretrained("THUDM/chatglm-6b-int4", trust_remote_code=True).half().cuda() ``` +**[2023/03/24]** 我们进一步提供了对Embedding量化后的模型，模型参数仅占用4.3 GB显存： +```python +model = AutoModel.from_pretrained("THUDM/chatglm-6b-int4-qe", trust_remote_code=True).half().cuda() +``` + + + ### CPU 部署 如果你没有 GPU 硬件的话，也可以在 CPU 上进行推理，但是推理速度会更慢。使用方法如下（需要大概 32GB 内存） ```python diff --git a/README_en.md b/README_en.md index ba9366d..3a8b17b 100644 --- a/README_en.md +++ b/README_en.md @@ -9,7 +9,7 @@ ChatGLM-6B uses technology similar to ChatGPT, optimized for Chinese QA and dial Try the [online demo](https://huggingface.co/spaces/ysharma/ChatGLM-6b_Gradio_Streaming) on Huggingface Spaces. ## Update -**[2023/03/23]** Add API deployment, thanks to [@LemonQu-GIT](https://github.com/LemonQu-GIT) +**[2023/03/23]** Add API deployment, thanks to [@LemonQu-GIT](https://github.com/LemonQu-GIT). Add embedding-quantized model [ChatGLM-6B-INT4-QE](https://huggingface.co/THUDM/chatglm-6b-int4-qe) **[2023/03/19]** Add streaming output function `stream_chat`, already applied in web and CLI demo. Fix Chinese punctuations in output. Add quantized model [ChatGLM-6B-INT4](https://huggingface.co/THUDM/chatglm-6b-int4). @@ -129,6 +129,11 @@ Model quantization brings a certain performance decline. After testing, ChatGLM- model = AutoModel.from_pretrained("THUDM/chatglm-6b-int4", trust_remote_code=True).half().cuda() ``` +**[2023/03/24]** We further provide an embedding-quantized model whose model parameters only cost 4.3GB GPU memory +```python +model = AutoModel.from_pretrained("THUDM/chatglm-6b-int4-qe", trust_remote_code=True).half().cuda() +``` + ### CPU Deployment If your computer is not equipped with GPU, you can also conduct inference on CPU, but the inference speed is slow (and taking about 32GB of memory): From 28665ade15d173ee60a6f2b904f70110912e99d4 Mon Sep 17 00:00:00 2001 From: duzx16 Date: Thu, 23 Mar 2023 22:53:05 +0800 Subject: [PATCH 12/54] Add support for M1 Mac --- README.md | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 14680b3..fa169aa 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ ChatGLM-6B 使用了和 ChatGPT 相似的技术，针对中文问答和对话进 *Read this in [English](README_en.md).* ## 更新信息 -**[2023/03/23]** 增加API部署（感谢 [@LemonQu-GIT](https://github.com/LemonQu-GIT)），增加Embedding量化模型[ChatGLM-6B-INT4-QE](https://huggingface.co/THUDM/chatglm-6b-int4-qe) +**[2023/03/23]** 增加API部署（感谢 [@LemonQu-GIT](https://github.com/LemonQu-GIT)）。增加Embedding量化模型[ChatGLM-6B-INT4-QE](https://huggingface.co/THUDM/chatglm-6b-int4-qe)。增加对基于Apple Silicon的Mac上GPU加速的支持。 **[2023/03/19]** 增加流式输出接口 `stream_chat`，已更新到网页版和命令行 Demo。修复输出中的中文标点。增加量化后的模型 [ChatGLM-6B-INT4](https://huggingface.co/THUDM/chatglm-6b-int4) @@ -94,7 +94,7 @@ python cli_demo.py 程序会在命令行中进行交互式的对话，在命令行中输入指示并回车即可生成回复，输入`clear`可以清空对话历史，输入`stop`终止程序。 -## API部署 +### API部署 首先需要安装额外的依赖`pip install fastapi uvicorn`，然后运行仓库中的[api.py](api.py)： ```shell python api.py @@ -153,6 +153,17 @@ model = AutoModel.from_pretrained("THUDM/chatglm-6b-int4",trust_remote_code=True 如果遇到了报错 `Could not find module 'nvcuda.dll'` 或者 `RuntimeError: Unknown platform: darwin` (MacOS) 的话请参考这个[Issue](https://github.com/THUDM/ChatGLM-6B/issues/6#issuecomment-1470060041). +### Mac 上的 GPU 加速 +对于搭载了Apple Silicon的Mac（以及MacBook），可以使用 MPS 后端来在 GPU 上运行 ChatGLM-6B。首先需要参考 Apple 的 [官方说明](https://developer.apple.com/metal/pytorch) 安装 PyTorch-Nightly。然后将模型仓库 clone 到本地 +```shell +git clone https://huggingface.co/THUDM/chatglm-6b +``` +将代码中的模型加载改为从本地加载，并使用 mps 后端 +```python +model = AutoModel.from_pretrained("your local path", trust_remote_code=True).half().to('mps') +``` +即可使用在 Mac 上使用 GPU 加速模型推理。 + ## ChatGLM-6B 示例 以下是一些使用 `web_demo.py` 得到的示例截图。更多 ChatGLM-6B 的可能，等待你来探索发现！ From acbc2e178a1aae2fcd44030e93c0de5733f5a1c8 Mon Sep 17 00:00:00 2001 From: ZhangErling <45256786+ZhangErling@users.noreply.github.com> Date: Fri, 24 Mar 2023 15:33:42 +0800 Subject: [PATCH 13/54] =?UTF-8?q?=E4=BF=AE=E5=A4=8Dapi=E6=9C=8D=E5=8A=A1?= =?UTF-8?q?=E5=90=AF=E5=8A=A8=E9=94=99=E8=AF=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 第31行【API】->【api】会因为大小写原因找不到API 第34行的下划线【chatglm_6b】->【chatglm-6b】会导致模型加载错误 --- api.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api.py b/api.py index 14a2d57..10f70b6 100644 --- a/api.py +++ b/api.py @@ -28,8 +28,8 @@ async def create_item(request: Request): if __name__ == '__main__': - uvicorn.run('API:app', host='0.0.0.0', port=8000, workers=1) + uvicorn.run('api:app', host='0.0.0.0', port=8000, workers=1) tokenizer = AutoTokenizer.from_pretrained("THUDM/chatglm-6b", trust_remote_code=True) -model = AutoModel.from_pretrained("THUDM/chatglm_6b", trust_remote_code=True).half().cuda() +model = AutoModel.from_pretrained("THUDM/chatglm-6b", trust_remote_code=True).half().cuda() model.eval() From 1047e446e5387aa06c856c95800f67beab8b80d4 Mon Sep 17 00:00:00 2001 From: Jianhao Mo Date: Fri, 24 Mar 2023 16:45:24 +0800 Subject: [PATCH 14/54] Update api.py bug fix --- api.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api.py b/api.py index 14a2d57..10f70b6 100644 --- a/api.py +++ b/api.py @@ -28,8 +28,8 @@ async def create_item(request: Request): if __name__ == '__main__': - uvicorn.run('API:app', host='0.0.0.0', port=8000, workers=1) + uvicorn.run('api:app', host='0.0.0.0', port=8000, workers=1) tokenizer = AutoTokenizer.from_pretrained("THUDM/chatglm-6b", trust_remote_code=True) -model = AutoModel.from_pretrained("THUDM/chatglm_6b", trust_remote_code=True).half().cuda() +model = AutoModel.from_pretrained("THUDM/chatglm-6b", trust_remote_code=True).half().cuda() model.eval() From bf39dac0670d4cbb734aaa4664001f1863cecf14 Mon Sep 17 00:00:00 2001 From: holk-h Date: Fri, 24 Mar 2023 18:34:09 +0800 Subject: [PATCH 15/54] Support stream out interruption by using Ctrl+C --- cli_demo.py | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/cli_demo.py b/cli_demo.py index 8a043fb..fea47fc 100644 --- a/cli_demo.py +++ b/cli_demo.py @@ -1,14 +1,15 @@ import os import platform +import signal from transformers import AutoTokenizer, AutoModel -tokenizer = AutoTokenizer.from_pretrained("THUDM/chatglm-6b", trust_remote_code=True) -model = AutoModel.from_pretrained("THUDM/chatglm-6b", trust_remote_code=True).half().cuda() +tokenizer = AutoTokenizer.from_pretrained("./model", trust_remote_code=True) +model = AutoModel.from_pretrained("./model", trust_remote_code=True).half().cuda() model = model.eval() os_name = platform.system() clear_command = 'cls' if os_name == 'Windows' else 'clear' - +stop_stream = False def build_prompt(history): prompt = "欢迎使用 ChatGLM-6B 模型，输入内容即可进行对话，clear 清空对话历史，stop 终止程序" @@ -17,9 +18,13 @@ def build_prompt(history): prompt += f"\n\nChatGLM-6B：{response}" return prompt +def signal_handler(signal, frame): + global stop_stream + stop_stream = True def main(): history = [] + global stop_stream print("欢迎使用 ChatGLM-6B 模型，输入内容即可进行对话，clear 清空对话历史，stop 终止程序") while True: query = input("\n用户：") @@ -32,10 +37,15 @@ def main(): continue count = 0 for response, history in model.stream_chat(tokenizer, query, history=history): - count += 1 - if count % 8 == 0: - os.system(clear_command) - print(build_prompt(history), flush=True) + if stop_stream: + stop_stream = False + break + else: + count += 1 + if count % 8 == 0: + os.system(clear_command) + print(build_prompt(history), flush=True) + signal.signal(signal.SIGINT,signal_handler) os.system(clear_command) print(build_prompt(history), flush=True) From ff260ffa51789ed9cebc4ba250e30d451a716d35 Mon Sep 17 00:00:00 2001 From: duzx16 Date: Sat, 25 Mar 2023 10:02:34 +0800 Subject: [PATCH 16/54] Update README.md --- README.md | 3 --- 1 file changed, 3 deletions(-) diff --git a/README.md b/README.md index fa169aa..58f3751 100644 --- a/README.md +++ b/README.md @@ -7,9 +7,6 @@ ChatGLM-6B 使用了和 ChatGPT 相似的技术，针对中文问答和对话进 不过，由于 ChatGLM-6B 的规模较小，目前已知其具有相当多的[**局限性**](#局限性)，如事实性/数学逻辑错误，可能生成有害/有偏见内容，较弱的上下文能力，自我认知混乱，以及对英文指示生成与中文指示完全矛盾的内容。请大家在使用前了解这些问题，以免产生误解。更大的基于1300亿参数[GLM-130B](https://github.com/THUDM/GLM-130B)的ChatGLM正在内测开发中。 -欢迎体验 Huggingface Spaces 上的[在线演示](https://huggingface.co/spaces/ysharma/ChatGLM-6b_Gradio_Streaming)。 - - *Read this in [English](README_en.md).* ## 更新信息 From 023c46a317537c9f2a6bdfc4916c77b24a00d868 Mon Sep 17 00:00:00 2001 From: littlepanda0716 Date: Sat, 25 Mar 2023 18:59:11 +0800 Subject: [PATCH 17/54] update api.py --- api.py | 28 +++++++++++++++++++++++++--- 1 file changed, 25 insertions(+), 3 deletions(-) diff --git a/api.py b/api.py index 10f70b6..4ad1db6 100644 --- a/api.py +++ b/api.py @@ -1,6 +1,19 @@ from fastapi import FastAPI, Request from transformers import AutoTokenizer, AutoModel import uvicorn, json, datetime +import torch + +DEVICE = "cuda" +DEVICE_ID = "0" +CUDA_DEVICE = f"{DEVICE}:{DEVICE_ID}" if DEVICE_ID else DEVICE + + +def torch_gc(): + if torch.cuda.is_available(): + with torch.cuda.device(CUDA_DEVICE): + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + app = FastAPI() @@ -13,7 +26,15 @@ async def create_item(request: Request): json_post_list = json.loads(json_post) prompt = json_post_list.get('prompt') history = json_post_list.get('history') - response, history = model.chat(tokenizer, prompt, history=history) + max_length = json_post_list.get('max_length') + top_p = json_post_list.get('top_p') + temperature = json_post_list.get('temperature') + response, history = model.chat(tokenizer, + prompt, + history=history, + max_length=max_length if max_length else 2048, + top_p=top_p if top_p else 0.7, + temperature=temperature if temperature else 0.95) now = datetime.datetime.now() time = now.strftime("%Y-%m-%d %H:%M:%S") answer = { @@ -24,12 +45,13 @@ async def create_item(request: Request): } log = "[" + time + "] " + '", prompt:"' + prompt + '", response:"' + repr(response) + '"' print(log) + torch_gc() return answer if __name__ == '__main__': - uvicorn.run('api:app', host='0.0.0.0', port=8000, workers=1) + uvicorn.run(app, host='0.0.0.0', port=8000, workers=1) tokenizer = AutoTokenizer.from_pretrained("THUDM/chatglm-6b", trust_remote_code=True) -model = AutoModel.from_pretrained("THUDM/chatglm-6b", trust_remote_code=True).half().cuda() +model = AutoModel.from_pretrained("THUDM/chatglm_6b", trust_remote_code=True).half().cuda() model.eval() From 9addb875ca503a37804fb82d55991b2f7392f8f9 Mon Sep 17 00:00:00 2001 From: duzx16 Date: Tue, 28 Mar 2023 16:58:38 +0800 Subject: [PATCH 18/54] Add links --- README.md | 6 ++++++ README_en.md | 7 +++++++ 2 files changed, 13 insertions(+) diff --git a/README.md b/README.md index 58f3751..075ce78 100644 --- a/README.md +++ b/README.md @@ -14,6 +14,12 @@ ChatGLM-6B 使用了和 ChatGPT 相似的技术，针对中文问答和对话进 **[2023/03/19]** 增加流式输出接口 `stream_chat`，已更新到网页版和命令行 Demo。修复输出中的中文标点。增加量化后的模型 [ChatGLM-6B-INT4](https://huggingface.co/THUDM/chatglm-6b-int4) +## 友情链接 +以下是部分基于本仓库开发的开源项目： +* [ChatGLM-MNN](https://github.com/wangzhaode/ChatGLM-MNN): 一个基于 MNN 的 ChatGLM-6B C++ 推理实现，支持根据显存大小动态分配计算任务给 GPU 和 CPU +* [ChatGLM-Tuning](https://github.com/mymusise/ChatGLM-Tuning): 基于 LoRA 对 ChatGLM-6B 进行微调 + +如果你有其他好的项目的话，欢迎参照上述格式添加到README中并提出 [PR](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/creating-a-pull-request-from-a-fork). ## 使用方式 diff --git a/README_en.md b/README_en.md index 3a8b17b..584e97f 100644 --- a/README_en.md +++ b/README_en.md @@ -13,6 +13,13 @@ Try the [online demo](https://huggingface.co/spaces/ysharma/ChatGLM-6b_Gradio_St **[2023/03/19]** Add streaming output function `stream_chat`, already applied in web and CLI demo. Fix Chinese punctuations in output. Add quantized model [ChatGLM-6B-INT4](https://huggingface.co/THUDM/chatglm-6b-int4). +## Projects +The following are some open source projects developed based on this repository: +* [ChatGLM-MNN](https://github.com/wangzhaode/ChatGLM-MNN): An [MNN](https://github.com/alibaba/MNN)-based implementation of ChatGLM-6B C++ inference, which supports dynamic allocation of computing tasks to GPU and CPU according to the size of GPU memory +* [ChatGLM-Tuning](https://github.com/mymusise/ChatGLM-Tuning): Fine-tuning ChatGLM-6B based on LoRA + +If you have other good projects, please refer to the above format to add to README and propose [PR](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/creating-a-pull-request-from-a-fork). + ## Getting Started ### Hardware Requirements From 1c6002f3f18496217c7c0d147ef34c291c6da65d Mon Sep 17 00:00:00 2001 From: duzx16 Date: Tue, 28 Mar 2023 19:29:41 +0800 Subject: [PATCH 19/54] Fix typo --- cli_demo.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cli_demo.py b/cli_demo.py index 8a043fb..0c6ed13 100644 --- a/cli_demo.py +++ b/cli_demo.py @@ -2,8 +2,8 @@ import os import platform from transformers import AutoTokenizer, AutoModel -tokenizer = AutoTokenizer.from_pretrained("THUDM/chatglm-6b", trust_remote_code=True) -model = AutoModel.from_pretrained("THUDM/chatglm-6b", trust_remote_code=True).half().cuda() +tokenizer = AutoTokenizer.from_pretrained("/Users/zhengxiaodu/Downloads/chatglm-6b", trust_remote_code=True) +model = AutoModel.from_pretrained("/Users/zhengxiaodu/Downloads/chatglm-6b", trust_remote_code=True).half().to("mps") model = model.eval() os_name = platform.system() From c6790a09f05ba6a23b073021bfa6c3df177442fc Mon Sep 17 00:00:00 2001 From: duzx16 Date: Tue, 28 Mar 2023 19:45:17 +0800 Subject: [PATCH 20/54] Fix typos Move model instantiation --- api.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/api.py b/api.py index 4ad1db6..693c70a 100644 --- a/api.py +++ b/api.py @@ -50,8 +50,7 @@ async def create_item(request: Request): if __name__ == '__main__': + tokenizer = AutoTokenizer.from_pretrained("THUDM/chatglm-6b", trust_remote_code=True) + model = AutoModel.from_pretrained("THUDM/chatglm-6b", trust_remote_code=True).half().cuda() + model.eval() uvicorn.run(app, host='0.0.0.0', port=8000, workers=1) - -tokenizer = AutoTokenizer.from_pretrained("THUDM/chatglm-6b", trust_remote_code=True) -model = AutoModel.from_pretrained("THUDM/chatglm_6b", trust_remote_code=True).half().cuda() -model.eval() From 6fc8141a9c9c57fe2e4897ff93491a9a61b0fb16 Mon Sep 17 00:00:00 2001 From: duzx16 Date: Tue, 28 Mar 2023 19:48:21 +0800 Subject: [PATCH 21/54] Revert "Fix typo" This reverts commit 1c6002f3f18496217c7c0d147ef34c291c6da65d. --- cli_demo.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cli_demo.py b/cli_demo.py index 0c6ed13..8a043fb 100644 --- a/cli_demo.py +++ b/cli_demo.py @@ -2,8 +2,8 @@ import os import platform from transformers import AutoTokenizer, AutoModel -tokenizer = AutoTokenizer.from_pretrained("/Users/zhengxiaodu/Downloads/chatglm-6b", trust_remote_code=True) -model = AutoModel.from_pretrained("/Users/zhengxiaodu/Downloads/chatglm-6b", trust_remote_code=True).half().to("mps") +tokenizer = AutoTokenizer.from_pretrained("THUDM/chatglm-6b", trust_remote_code=True) +model = AutoModel.from_pretrained("THUDM/chatglm-6b", trust_remote_code=True).half().cuda() model = model.eval() os_name = platform.system() From 343e7bc7b6126718f10dc57ef3c911958c4b273b Mon Sep 17 00:00:00 2001 From: duzx16 Date: Tue, 28 Mar 2023 19:52:32 +0800 Subject: [PATCH 22/54] Fix model path --- cli_demo.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/cli_demo.py b/cli_demo.py index fea47fc..1c3ff2b 100644 --- a/cli_demo.py +++ b/cli_demo.py @@ -3,14 +3,15 @@ import platform import signal from transformers import AutoTokenizer, AutoModel -tokenizer = AutoTokenizer.from_pretrained("./model", trust_remote_code=True) -model = AutoModel.from_pretrained("./model", trust_remote_code=True).half().cuda() +tokenizer = AutoTokenizer.from_pretrained("THUDM/chatglm-6b", trust_remote_code=True) +model = AutoModel.from_pretrained("THUDM/chatglm-6b", trust_remote_code=True).half().cuda() model = model.eval() os_name = platform.system() clear_command = 'cls' if os_name == 'Windows' else 'clear' stop_stream = False + def build_prompt(history): prompt = "欢迎使用 ChatGLM-6B 模型，输入内容即可进行对话，clear 清空对话历史，stop 终止程序" for query, response in history: @@ -18,10 +19,12 @@ def build_prompt(history): prompt += f"\n\nChatGLM-6B：{response}" return prompt + def signal_handler(signal, frame): global stop_stream stop_stream = True + def main(): history = [] global stop_stream @@ -45,7 +48,7 @@ def main(): if count % 8 == 0: os.system(clear_command) print(build_prompt(history), flush=True) - signal.signal(signal.SIGINT,signal_handler) + signal.signal(signal.SIGINT, signal_handler) os.system(clear_command) print(build_prompt(history), flush=True) From 7d7d87c4bd26d84485e09d284437fc0682deb45f Mon Sep 17 00:00:00 2001 From: duzx16 Date: Tue, 28 Mar 2023 21:18:59 +0800 Subject: [PATCH 23/54] Update README --- README.md | 2 +- README_en.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 075ce78..2f59ad6 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,7 @@ ChatGLM-6B 使用了和 ChatGPT 相似的技术，针对中文问答和对话进 ## 友情链接 以下是部分基于本仓库开发的开源项目： -* [ChatGLM-MNN](https://github.com/wangzhaode/ChatGLM-MNN): 一个基于 MNN 的 ChatGLM-6B C++ 推理实现，支持根据显存大小动态分配计算任务给 GPU 和 CPU +* [ChatGLM-MNN](https://github.com/wangzhaode/ChatGLM-MNN): 一个基于 MNN 的 ChatGLM-6B C++ 推理实现，支持根据显存大小自动分配计算任务给 GPU 和 CPU * [ChatGLM-Tuning](https://github.com/mymusise/ChatGLM-Tuning): 基于 LoRA 对 ChatGLM-6B 进行微调 如果你有其他好的项目的话，欢迎参照上述格式添加到README中并提出 [PR](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/creating-a-pull-request-from-a-fork). diff --git a/README_en.md b/README_en.md index 584e97f..7b84f9c 100644 --- a/README_en.md +++ b/README_en.md @@ -15,7 +15,7 @@ Try the [online demo](https://huggingface.co/spaces/ysharma/ChatGLM-6b_Gradio_St ## Projects The following are some open source projects developed based on this repository: -* [ChatGLM-MNN](https://github.com/wangzhaode/ChatGLM-MNN): An [MNN](https://github.com/alibaba/MNN)-based implementation of ChatGLM-6B C++ inference, which supports dynamic allocation of computing tasks to GPU and CPU according to the size of GPU memory +* [ChatGLM-MNN](https://github.com/wangzhaode/ChatGLM-MNN): An [MNN](https://github.com/alibaba/MNN)-based implementation of ChatGLM-6B C++ inference, which supports automatic allocation of computing tasks to GPU and CPU according to the size of GPU memory * [ChatGLM-Tuning](https://github.com/mymusise/ChatGLM-Tuning): Fine-tuning ChatGLM-6B based on LoRA If you have other good projects, please refer to the above format to add to README and propose [PR](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/creating-a-pull-request-from-a-fork). From fc55c075fe8a2c403917d2c7b04d3f0ef3b0e0f2 Mon Sep 17 00:00:00 2001 From: duzx16 Date: Tue, 28 Mar 2023 21:35:52 +0800 Subject: [PATCH 24/54] Update README --- README.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 2f59ad6..814277d 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,10 @@ ChatGLM-6B 使用了和 ChatGPT 相似的技术，针对中文问答和对话进 * [ChatGLM-MNN](https://github.com/wangzhaode/ChatGLM-MNN): 一个基于 MNN 的 ChatGLM-6B C++ 推理实现，支持根据显存大小自动分配计算任务给 GPU 和 CPU * [ChatGLM-Tuning](https://github.com/mymusise/ChatGLM-Tuning): 基于 LoRA 对 ChatGLM-6B 进行微调 -如果你有其他好的项目的话，欢迎参照上述格式添加到README中并提出 [PR](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/creating-a-pull-request-from-a-fork). +以下是部分针对本项目的教程/文档： +* [Windows部署文档](https://github.com/ZhangErling/ChatGLM-6B/blob/main/deployment_windows.md) + +如果你有其他好的项目/教程的话，欢迎参照上述格式添加到README中并提出 [PR](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/creating-a-pull-request-from-a-fork). ## 使用方式 From 323ce7c86530ea3be87f70eabcee27ee38ec29e9 Mon Sep 17 00:00:00 2001 From: duzx16 Date: Wed, 29 Mar 2023 23:24:33 +0800 Subject: [PATCH 25/54] Add instructions for installing Git LFS --- README.md | 3 ++- README_en.md | 18 ++++++++++++++++-- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 814277d..ec90fb8 100644 --- a/README.md +++ b/README.md @@ -160,8 +160,9 @@ model = AutoModel.from_pretrained("THUDM/chatglm-6b-int4",trust_remote_code=True 如果遇到了报错 `Could not find module 'nvcuda.dll'` 或者 `RuntimeError: Unknown platform: darwin` (MacOS) 的话请参考这个[Issue](https://github.com/THUDM/ChatGLM-6B/issues/6#issuecomment-1470060041). ### Mac 上的 GPU 加速 -对于搭载了Apple Silicon的Mac（以及MacBook），可以使用 MPS 后端来在 GPU 上运行 ChatGLM-6B。首先需要参考 Apple 的 [官方说明](https://developer.apple.com/metal/pytorch) 安装 PyTorch-Nightly。然后将模型仓库 clone 到本地 +对于搭载了Apple Silicon的Mac（以及MacBook），可以使用 MPS 后端来在 GPU 上运行 ChatGLM-6B。首先需要参考 Apple 的 [官方说明](https://developer.apple.com/metal/pytorch) 安装 PyTorch-Nightly。然后将模型仓库 clone 到本地（需要先[安装Git LFS](https://docs.github.com/zh/repositories/working-with-files/managing-large-files/installing-git-large-file-storage)） ```shell +git lfs install git clone https://huggingface.co/THUDM/chatglm-6b ``` 将代码中的模型加载改为从本地加载，并使用 mps 后端 diff --git a/README_en.md b/README_en.md index 7b84f9c..b5b4b62 100644 --- a/README_en.md +++ b/README_en.md @@ -9,7 +9,7 @@ ChatGLM-6B uses technology similar to ChatGPT, optimized for Chinese QA and dial Try the [online demo](https://huggingface.co/spaces/ysharma/ChatGLM-6b_Gradio_Streaming) on Huggingface Spaces. ## Update -**[2023/03/23]** Add API deployment, thanks to [@LemonQu-GIT](https://github.com/LemonQu-GIT). Add embedding-quantized model [ChatGLM-6B-INT4-QE](https://huggingface.co/THUDM/chatglm-6b-int4-qe) +**[2023/03/23]** Add API deployment, thanks to [@LemonQu-GIT](https://github.com/LemonQu-GIT). Add embedding-quantized model [ChatGLM-6B-INT4-QE](https://huggingface.co/THUDM/chatglm-6b-int4-qe). Add support for GPU inference on Mac with Apple Silicon. **[2023/03/19]** Add streaming output function `stream_chat`, already applied in web and CLI demo. Fix Chinese punctuations in output. Add quantized model [ChatGLM-6B-INT4](https://huggingface.co/THUDM/chatglm-6b-int4). @@ -154,7 +154,21 @@ model = AutoModel.from_pretrained("THUDM/chatglm-6b", trust_remote_code=True).fl model = AutoModel.from_pretrained("THUDM/chatglm-6b-int4", trust_remote_code=True).float() ``` -**For Mac users**: if your encounter the error `RuntimeError: Unknown platform: darwin`, please refer to this [Issue](https://github.com/THUDM/ChatGLM-6B/issues/6#issuecomment-1470060041). +If your encounter the error `Could not find module 'nvcuda.dll'` or `RuntimeError: Unknown platform: darwin`(MacOS), please refer to this [Issue](https://github.com/THUDM/ChatGLM-6B/issues/6#issuecomment-1470060041). + +### GPU Inference on Mac +For Macs (and MacBooks) with Apple Silicon, it is possible to use the MPS backend to run ChatGLM-6B on the GPU. First, you need to refer to Apple's [official instructions](https://developer.apple.com/metal/pytorch) to install PyTorch-Nightly. Then clone the model repository locally (you need to [install Git LFS](https://docs.github.com/zh/repositories/working-with-files/managing-large-files/installing-git-large-file-storage)） +```shell +git lfs install +git clone https://huggingface.co/THUDM/chatglm-6b +``` +Change the code to load the model from your local path, and use the mps backend: +```python +model = AutoModel.from_pretrained("your local path", trust_remote_code=True).half().to('mps') +``` +Then you can use GPU-accelerated model inference on Mac. + + ## ChatGLM-6B Examples From 32d625463ce53ac5986c1ea4974dda3d7687f1a0 Mon Sep 17 00:00:00 2001 From: Shaw Date: Thu, 30 Mar 2023 11:49:07 +0800 Subject: [PATCH 26/54] Update README.md --- README.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index ec90fb8..0bcfc06 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,15 @@ # ChatGLM-6B +

+ 🌐 Blog • 🤗 HF Repo • 🐦 Twitter • 📃 [GLM@ACL 22] [GitHub] • 📃 [GLM-130B@ICLR 23] [GitHub]
+

+ ## 介绍 ChatGLM-6B 是一个开源的、支持中英双语的对话语言模型，基于 [General Language Model (GLM)](https://github.com/THUDM/GLM) 架构，具有 62 亿参数。结合模型量化技术，用户可以在消费级的显卡上进行本地部署（INT4 量化级别下最低只需 6GB 显存）。 ChatGLM-6B 使用了和 ChatGPT 相似的技术，针对中文问答和对话进行了优化。经过约 1T 标识符的中英双语训练，辅以监督微调、反馈自助、人类反馈强化学习等技术的加持，62 亿参数的 ChatGLM-6B 已经能生成相当符合人类偏好的回答。更多信息请参考我们的[博客](https://chatglm.cn/blog)。 -不过，由于 ChatGLM-6B 的规模较小，目前已知其具有相当多的[**局限性**](#局限性)，如事实性/数学逻辑错误，可能生成有害/有偏见内容，较弱的上下文能力，自我认知混乱，以及对英文指示生成与中文指示完全矛盾的内容。请大家在使用前了解这些问题，以免产生误解。更大的基于1300亿参数[GLM-130B](https://github.com/THUDM/GLM-130B)的ChatGLM正在内测开发中。 +不过，由于 ChatGLM-6B 的规模较小，目前已知其具有相当多的[**局限性**](#局限性)，如事实性/数学逻辑错误，可能生成有害/有偏见内容，较弱的上下文能力，自我认知混乱，以及对英文指示生成与中文指示完全矛盾的内容。请大家在使用前了解这些问题，以免产生误解。更大的基于1300亿参数 [GLM-130B](https://github.com/THUDM/GLM-130B) 的ChatGLM正在内测开发中。 *Read this in [English](README_en.md).* From ee7fa65ebd1fcac780f30d8f25acb241e27304bc Mon Sep 17 00:00:00 2001 From: Shaw Date: Thu, 30 Mar 2023 11:49:53 +0800 Subject: [PATCH 27/54] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 0bcfc06..2c6f083 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ ChatGLM-6B 是一个开源的、支持中英双语的对话语言模型，基于 [General Language Model (GLM)](https://github.com/THUDM/GLM) 架构，具有 62 亿参数。结合模型量化技术，用户可以在消费级的显卡上进行本地部署（INT4 量化级别下最低只需 6GB 显存）。 ChatGLM-6B 使用了和 ChatGPT 相似的技术，针对中文问答和对话进行了优化。经过约 1T 标识符的中英双语训练，辅以监督微调、反馈自助、人类反馈强化学习等技术的加持，62 亿参数的 ChatGLM-6B 已经能生成相当符合人类偏好的回答。更多信息请参考我们的[博客](https://chatglm.cn/blog)。 -不过，由于 ChatGLM-6B 的规模较小，目前已知其具有相当多的[**局限性**](#局限性)，如事实性/数学逻辑错误，可能生成有害/有偏见内容，较弱的上下文能力，自我认知混乱，以及对英文指示生成与中文指示完全矛盾的内容。请大家在使用前了解这些问题，以免产生误解。更大的基于1300亿参数 [GLM-130B](https://github.com/THUDM/GLM-130B) 的ChatGLM正在内测开发中。 +不过，由于 ChatGLM-6B 的规模较小，目前已知其具有相当多的[**局限性**](#局限性)，如事实性/数学逻辑错误，可能生成有害/有偏见内容，较弱的上下文能力，自我认知混乱，以及对英文指示生成与中文指示完全矛盾的内容。请大家在使用前了解这些问题，以免产生误解。更大的基于1300亿参数 [GLM-130B](https://github.com/THUDM/GLM-130B) 的 ChatGLM 正在内测开发中。 *Read this in [English](README_en.md).* From 968a30672ab90b6d0f3b6be9a098db567b34e06f Mon Sep 17 00:00:00 2001 From: duzx16 Date: Fri, 31 Mar 2023 10:43:55 +0800 Subject: [PATCH 28/54] Add P-Tuning v2 --- README.md | 25 ++- ptuning/README.md | 70 +++++++ ptuning/arguments.py | 217 +++++++++++++++++++++ ptuning/evaluate.sh | 20 ++ ptuning/main.py | 389 +++++++++++++++++++++++++++++++++++++ ptuning/train.sh | 26 +++ ptuning/trainer_seq2seq.py | 245 +++++++++++++++++++++++ 7 files changed, 982 insertions(+), 10 deletions(-) create mode 100644 ptuning/README.md create mode 100644 ptuning/arguments.py create mode 100644 ptuning/evaluate.sh create mode 100644 ptuning/main.py create mode 100644 ptuning/train.sh create mode 100644 ptuning/trainer_seq2seq.py diff --git a/README.md b/README.md index ec90fb8..ff4383b 100644 --- a/README.md +++ b/README.md @@ -10,20 +10,12 @@ ChatGLM-6B 使用了和 ChatGPT 相似的技术，针对中文问答和对话进 *Read this in [English](README_en.md).* ## 更新信息 +**[2023/03/31]** 增加基于 P-Tuning-v2 的微调实现，最低只需 8GB 显存即可进行模型微调。详见[模型微调](ptuning/README.md)。 + **[2023/03/23]** 增加API部署（感谢 [@LemonQu-GIT](https://github.com/LemonQu-GIT)）。增加Embedding量化模型[ChatGLM-6B-INT4-QE](https://huggingface.co/THUDM/chatglm-6b-int4-qe)。增加对基于Apple Silicon的Mac上GPU加速的支持。 **[2023/03/19]** 增加流式输出接口 `stream_chat`，已更新到网页版和命令行 Demo。修复输出中的中文标点。增加量化后的模型 [ChatGLM-6B-INT4](https://huggingface.co/THUDM/chatglm-6b-int4) -## 友情链接 -以下是部分基于本仓库开发的开源项目： -* [ChatGLM-MNN](https://github.com/wangzhaode/ChatGLM-MNN): 一个基于 MNN 的 ChatGLM-6B C++ 推理实现，支持根据显存大小自动分配计算任务给 GPU 和 CPU -* [ChatGLM-Tuning](https://github.com/mymusise/ChatGLM-Tuning): 基于 LoRA 对 ChatGLM-6B 进行微调 - -以下是部分针对本项目的教程/文档： -* [Windows部署文档](https://github.com/ZhangErling/ChatGLM-6B/blob/main/deployment_windows.md) - -如果你有其他好的项目/教程的话，欢迎参照上述格式添加到README中并提出 [PR](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/creating-a-pull-request-from-a-fork). - ## 使用方式 ### 硬件需求 @@ -171,6 +163,9 @@ model = AutoModel.from_pretrained("your local path", trust_remote_code=True).hal ``` 即可使用在 Mac 上使用 GPU 加速模型推理。 +## 模型微调 +详见 [ptuning/README.md](ptuning/README.md)。 + ## ChatGLM-6B 示例 以下是一些使用 `web_demo.py` 得到的示例截图。更多 ChatGLM-6B 的可能，等待你来探索发现！ @@ -259,6 +254,16 @@ model = AutoModel.from_pretrained("your local path", trust_remote_code=True).hal 本仓库的代码依照 [Apache-2.0](LICENSE) 协议开源，ChatGLM-6B 模型的权重的使用则需要遵循 [Model License](MODEL_LICENSE)。 +## 友情链接 +以下是部分基于本仓库开发的开源项目： +* [ChatGLM-MNN](https://github.com/wangzhaode/ChatGLM-MNN): 一个基于 MNN 的 ChatGLM-6B C++ 推理实现，支持根据显存大小自动分配计算任务给 GPU 和 CPU +* [ChatGLM-Tuning](https://github.com/mymusise/ChatGLM-Tuning): 基于 LoRA 对 ChatGLM-6B 进行微调 + +以下是部分针对本项目的教程/文档： +* [Windows部署文档](https://github.com/ZhangErling/ChatGLM-6B/blob/main/deployment_windows.md) + +如果你有其他好的项目/教程的话，欢迎参照上述格式添加到README中并提出 [PR](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/creating-a-pull-request-from-a-fork). + ## 引用 如果你觉得我们的工作有帮助的话，请考虑引用下列论文 diff --git a/ptuning/README.md b/ptuning/README.md new file mode 100644 index 0000000..1fb0ea6 --- /dev/null +++ b/ptuning/README.md @@ -0,0 +1,70 @@ +# ChatGLM-6B-PT +本仓库实现了对于 ChatGLM-6B 模型基于 [P-Tuning v2](https://github.com/THUDM/P-tuning-v2) 的微调。P-Tuning v2将需要微调的参数量减少到原来的0.1%，再通过模型量化、Gradient Checkpoint等方法，最低只需要 8GB 显存即可运行。 + +下面以 [ADGEN](https://aclanthology.org/D19-1321.pdf) (广告生成) 数据集为例介绍代码的使用方法。 + +## 软件依赖 +除 ChatGLM-6B 的依赖之外，还需要按照以下依赖 +``` +pip install rouge_chinese nltk jieba datasets +``` +## 使用方法 + +### 下载数据集 +ADGEN 数据集任务为根据输入（content）生成一段广告词（summary）。 + +```json +{ + "content": "类型#上衣*版型#宽松*版型#显瘦*图案#线条*衣样式#衬衫*衣袖型#泡泡袖*衣款式#抽绳", + "summary": "这件衬衫的款式非常的宽松，利落的线条可以很好的隐藏身材上的小缺点，穿在身上有着很好的显瘦效果。领口装饰了一个可爱的抽绳，漂亮的绳结展现出了十足的个性，配合时尚的泡泡袖型，尽显女性甜美可爱的气息。" +} +``` + +从 [Google Drive](https://drive.google.com/file/d/13_vf0xRTQsyneRKdD1bZIr93vBGOczrk/view?usp=sharing) 或者 [Tsinghua Cloud]() 下载处理好的 ADGEN数据集，将解压后的 `AdvertiseGen` 目录放到本目录下。 + +### 训练 +运行以下指令进行训练： +```shell +bash train.sh +``` +`train.sh` 中的`PRE_SEQ_LEN`和 `LR` 分别是 soft prompt 长度和训练的学习率，可以进行调节以取得最佳的效果。 + +### 推理 + +将`evaluate.sh`中的`CHECKPOINT`更改为训练时保存的checkpoint名称，运行以下指令进行模型推理和评测： +```shell +bash evaluate.sh +``` + +评测指标为中文 Rouge score 和 BLEU-4。生成的结果保存在 +`./output/adgen-chatglm-6b-pt-8-1e-2/generated_predictions.txt`。 + +### 例子 +#### 示例1 +* Input: 类型#上衣\*材质#牛仔布\*颜色#白色\*风格#简约\*图案#刺绣\*衣样式#外套\*衣款式#破洞 +* Label: 简约而不简单的牛仔外套,白色的衣身十分百搭。衣身多处有做旧破洞设计,打破单调乏味,增加一丝造型看点。衣身后背处有趣味刺绣装饰,丰富层次感,彰显别样时尚。 +* 微调前Output: 这件上衣的材质是牛仔布,颜色是白色,风格是简约,图案是刺绣,衣样式是外套,衣款式是破洞。 +* 微调后Output: 这是一款简约的牛仔外套,破洞设计,将牛仔布破洞,带来一种随意与个性。破洞的牛仔外套,展现出时尚气息,带来一种休闲感。同时,刺绣图案,让整件外套更加立体。 + +#### 示例2 + +* Input: 类型#裙\*版型#显瘦\*风格#文艺\*风格#简约\*图案#印花\*图案#撞色\*裙下摆#压褶\*裙长#连衣裙\*裙领型#圆领 +* Label: 文艺个性的印花连衣裙,藏青色底蕴,低调又大气,撞色太阳花分布整个裙身,绚丽而美好,带来时尚减龄的气质。基础款的舒适圆领,简约不失大方,勾勒精致脸庞。领后是一粒包布扣固定,穿脱十分方便。前片立体的打褶设计,搭配后片压褶的做工,增添层次和空间感,显瘦又有型。 + +* 微调前Output: 类型#裙*版型#显瘦*风格#文艺*风格#简约*图案#印花*图案#撞色*裙下摆#压褶*裙长#连衣裙*裙领型#圆领 1\. 连衣裙:简约风格,裙长为膝盖以上,裙领型为圆领。2\. 裙下摆:压褶设计,使裙摆呈现出流畅的褶皱效果。3\. 裙领型:裙领型为圆领,使穿上连衣裙后更加有型。4\. 版型:采用显瘦设计,让连衣裙看起来更加苗条。5\. 风格:文艺风格,让连衣裙更加有内涵和品味。6\. 图案:印花设计,在连衣裙上印有独特的图案。7\. 撞色:采用撞色设计,让连衣裙在色彩上更加鲜明、富有层次感。 +* 微调后Output: 这是一款文艺范的连衣裙,以印花为元素,采用简约的印花,既能够突出文艺气质,又能够展现简约风。在印花的同时又有领子和裙摆的压褶设计,更加凸显文艺气质。简约而不会过于单调,搭配出街,穿着十分舒适。 + +## 使用自己的数据集 +修改 `train.sh` 和 `evaluate.sh` 中的 `train_file`、`validation_file`和`test_file`为你自己的json格式数据集路径，并将`prompt_column`和`response_column`改为json文件中输入文本和输出文本对应的key。 + +## 引用 + +``` +@inproceedings{liu2022p, + title={P-tuning: Prompt tuning can be comparable to fine-tuning across scales and tasks}, + author={Liu, Xiao and Ji, Kaixuan and Fu, Yicheng and Tam, Weng and Du, Zhengxiao and Yang, Zhilin and Tang, Jie}, + booktitle={Proceedings of the 60th Annual Meeting of the Association for Computational Linguistics (Volume 2: Short Papers)}, + pages={61--68}, + year={2022} +} +``` diff --git a/ptuning/arguments.py b/ptuning/arguments.py new file mode 100644 index 0000000..1c61f97 --- /dev/null +++ b/ptuning/arguments.py @@ -0,0 +1,217 @@ +from dataclasses import dataclass, field +from typing import Optional + + +@dataclass +class ModelArguments: + """ + Arguments pertaining to which model/config/tokenizer we are going to fine-tune from. + """ + + model_name_or_path: str = field( + metadata={"help": "Path to pretrained model or model identifier from huggingface.co/models"} + ) + config_name: Optional[str] = field( + default=None, metadata={"help": "Pretrained config name or path if not the same as model_name"} + ) + tokenizer_name: Optional[str] = field( + default=None, metadata={"help": "Pretrained tokenizer name or path if not the same as model_name"} + ) + cache_dir: Optional[str] = field( + default=None, + metadata={"help": "Where to store the pretrained models downloaded from huggingface.co"}, + ) + use_fast_tokenizer: bool = field( + default=True, + metadata={"help": "Whether to use one of the fast tokenizer (backed by the tokenizers library) or not."}, + ) + model_revision: str = field( + default="main", + metadata={"help": "The specific model version to use (can be a branch name, tag name or commit id)."}, + ) + use_auth_token: bool = field( + default=False, + metadata={ + "help": ( + "Will use the token generated when running `huggingface-cli login` (necessary to use this script " + "with private models)." + ) + }, + ) + resize_position_embeddings: Optional[bool] = field( + default=None, + metadata={ + "help": ( + "Whether to automatically resize the position embeddings if `max_source_length` exceeds " + "the model's position embeddings." + ) + }, + ) + quantization_bit: Optional[int] = field( + default=None + ) + pre_seq_len: Optional[int] = field( + default=None + ) + prefix_projection: bool = field( + default=False + ) + + +@dataclass +class DataTrainingArguments: + """ + Arguments pertaining to what data we are going to input our model for training and eval. + """ + + lang: Optional[str] = field(default=None, metadata={"help": "Language id for summarization."}) + + dataset_name: Optional[str] = field( + default=None, metadata={"help": "The name of the dataset to use (via the datasets library)."} + ) + dataset_config_name: Optional[str] = field( + default=None, metadata={"help": "The configuration name of the dataset to use (via the datasets library)."} + ) + prompt_column: Optional[str] = field( + default=None, + metadata={"help": "The name of the column in the datasets containing the full texts (for summarization)."}, + ) + response_column: Optional[str] = field( + default=None, + metadata={"help": "The name of the column in the datasets containing the summaries (for summarization)."}, + ) + train_file: Optional[str] = field( + default=None, metadata={"help": "The input training data file (a jsonlines or csv file)."} + ) + validation_file: Optional[str] = field( + default=None, + metadata={ + "help": ( + "An optional input evaluation data file to evaluate the metrics (rouge) on (a jsonlines or csv file)." + ) + }, + ) + test_file: Optional[str] = field( + default=None, + metadata={ + "help": "An optional input test data file to evaluate the metrics (rouge) on (a jsonlines or csv file)." + }, + ) + overwrite_cache: bool = field( + default=False, metadata={"help": "Overwrite the cached training and evaluation sets"} + ) + preprocessing_num_workers: Optional[int] = field( + default=None, + metadata={"help": "The number of processes to use for the preprocessing."}, + ) + max_source_length: Optional[int] = field( + default=1024, + metadata={ + "help": ( + "The maximum total input sequence length after tokenization. Sequences longer " + "than this will be truncated, sequences shorter will be padded." + ) + }, + ) + max_target_length: Optional[int] = field( + default=128, + metadata={ + "help": ( + "The maximum total sequence length for target text after tokenization. Sequences longer " + "than this will be truncated, sequences shorter will be padded." + ) + }, + ) + val_max_target_length: Optional[int] = field( + default=None, + metadata={ + "help": ( + "The maximum total sequence length for validation target text after tokenization. Sequences longer " + "than this will be truncated, sequences shorter will be padded. Will default to `max_target_length`." + "This argument is also used to override the ``max_length`` param of ``model.generate``, which is used " + "during ``evaluate`` and ``predict``." + ) + }, + ) + pad_to_max_length: bool = field( + default=False, + metadata={ + "help": ( + "Whether to pad all samples to model maximum sentence length. " + "If False, will pad the samples dynamically when batching to the maximum length in the batch. More " + "efficient on GPU but very bad for TPU." + ) + }, + ) + max_train_samples: Optional[int] = field( + default=None, + metadata={ + "help": ( + "For debugging purposes or quicker training, truncate the number of training examples to this " + "value if set." + ) + }, + ) + max_eval_samples: Optional[int] = field( + default=None, + metadata={ + "help": ( + "For debugging purposes or quicker training, truncate the number of evaluation examples to this " + "value if set." + ) + }, + ) + max_predict_samples: Optional[int] = field( + default=None, + metadata={ + "help": ( + "For debugging purposes or quicker training, truncate the number of prediction examples to this " + "value if set." + ) + }, + ) + num_beams: Optional[int] = field( + default=None, + metadata={ + "help": ( + "Number of beams to use for evaluation. This argument will be passed to ``model.generate``, " + "which is used during ``evaluate`` and ``predict``." + ) + }, + ) + ignore_pad_token_for_loss: bool = field( + default=True, + metadata={ + "help": "Whether to ignore the tokens corresponding to padded labels in the loss computation or not." + }, + ) + source_prefix: Optional[str] = field( + default="", metadata={"help": "A prefix to add before every source text (useful for T5 models)."} + ) + + forced_bos_token: Optional[str] = field( + default=None, + metadata={ + "help": ( + "The token to force as the first generated token after the decoder_start_token_id." + "Useful for multilingual models like mBART where the first generated token" + "needs to be the target language token (Usually it is the target language token)" + ) + }, + ) + + + + def __post_init__(self): + if self.dataset_name is None and self.train_file is None and self.validation_file is None: + raise ValueError("Need either a dataset name or a training/validation file.") + else: + if self.train_file is not None: + extension = self.train_file.split(".")[-1] + assert extension in ["csv", "json"], "`train_file` should be a csv or a json file." + if self.validation_file is not None: + extension = self.validation_file.split(".")[-1] + assert extension in ["csv", "json"], "`validation_file` should be a csv or a json file." + if self.val_max_target_length is None: + self.val_max_target_length = self.max_target_length + diff --git a/ptuning/evaluate.sh b/ptuning/evaluate.sh new file mode 100644 index 0000000..db2a8c1 --- /dev/null +++ b/ptuning/evaluate.sh @@ -0,0 +1,20 @@ +PRE_SEQ_LEN=8 +CHECKPOINT=adgen-chatglm-6b-pt-8-1e-2 +STEP=3000 + +CUDA_VISIBLE_DEVICES=0 python3 main.py \ + --do_predict \ + --test_file AdvertiseGen/dev.json \ + --overwrite_cache \ + --prompt_column content \ + --response_column summary \ + --model_name_or_path ./output/$CHECKPOINT/checkpoint-$STEP \ + --output_dir ./output/$CHECKPOINT \ + --overwrite_output_dir \ + --max_source_length 64 \ + --max_target_length 64 \ + --per_device_eval_batch_size 1 \ + --predict_with_generate \ + --max_predict_samples 10 \ + --pre_seq_len $PRE_SEQ_LEN \ + --quantization_bit 4 diff --git a/ptuning/main.py b/ptuning/main.py new file mode 100644 index 0000000..d82fccc --- /dev/null +++ b/ptuning/main.py @@ -0,0 +1,389 @@ +#!/usr/bin/env python +# coding=utf-8 +# Copyright 2021 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Fine-tuning the library models for sequence to sequence. +""" +# You can also adapt this script on your own sequence to sequence task. Pointers for this are left as comments. + +import logging +import os +import sys +import json + +import numpy as np +from datasets import load_dataset +import jieba +from rouge_chinese import Rouge +from nltk.translate.bleu_score import sentence_bleu + +import transformers +from transformers import ( + AutoConfig, + AutoModel, + AutoTokenizer, + AutoTokenizer, + DataCollatorForSeq2Seq, + HfArgumentParser, + Seq2SeqTrainingArguments, + set_seed, +) +from trainer_seq2seq import Seq2SeqTrainer + +from arguments import ModelArguments, DataTrainingArguments + +logger = logging.getLogger(__name__) + +def main(): + + parser = HfArgumentParser((ModelArguments, DataTrainingArguments, Seq2SeqTrainingArguments)) + if len(sys.argv) == 2 and sys.argv[1].endswith(".json"): + # If we pass only one argument to the script and it's the path to a json file, + # let's parse it to get our arguments. + model_args, data_args, training_args = parser.parse_json_file(json_file=os.path.abspath(sys.argv[1])) + else: + model_args, data_args, training_args = parser.parse_args_into_dataclasses() + + # Setup logging + logging.basicConfig( + format="%(asctime)s - %(levelname)s - %(name)s - %(message)s", + datefmt="%m/%d/%Y %H:%M:%S", + handlers=[logging.StreamHandler(sys.stdout)], + ) + + if training_args.should_log: + # The default of training_args.log_level is passive, so we set log level at info here to have that default. + transformers.utils.logging.set_verbosity_info() + + log_level = training_args.get_process_log_level() + logger.setLevel(log_level) + # datasets.utils.logging.set_verbosity(log_level) + transformers.utils.logging.set_verbosity(log_level) + transformers.utils.logging.enable_default_handler() + transformers.utils.logging.enable_explicit_format() + + # Log on each process the small summary: + logger.warning( + f"Process rank: {training_args.local_rank}, device: {training_args.device}, n_gpu: {training_args.n_gpu}" + + f"distributed training: {bool(training_args.local_rank != -1)}, 16-bits training: {training_args.fp16}" + ) + logger.info(f"Training/evaluation parameters {training_args}") + + # Set seed before initializing model. + set_seed(training_args.seed) + + # Load dataset + data_files = {} + if data_args.train_file is not None: + data_files["train"] = data_args.train_file + extension = data_args.train_file.split(".")[-1] + if data_args.validation_file is not None: + data_files["validation"] = data_args.validation_file + extension = data_args.validation_file.split(".")[-1] + if data_args.test_file is not None: + data_files["test"] = data_args.test_file + extension = data_args.test_file.split(".")[-1] + + raw_datasets = load_dataset( + extension, + data_files=data_files, + cache_dir=model_args.cache_dir, + use_auth_token=True if model_args.use_auth_token else None, + ) + + # Load pretrained model and tokenizer + config = AutoConfig.from_pretrained(model_args.model_name_or_path, trust_remote_code=True) + config.pre_seq_len = model_args.pre_seq_len + config.prefix_projection = model_args.prefix_projection + + tokenizer = AutoTokenizer.from_pretrained(model_args.model_name_or_path, trust_remote_code=True) + + model = AutoModel.from_pretrained(model_args.model_name_or_path, config=config, revision=True, trust_remote_code=True) + + model = model.half() + if model_args.quantization_bit is not None: + print(f"Quantized to {model_args.quantization_bit} bit") + model = model.quantize(model_args.quantization_bit) + model.transformer.prefix_encoder.float() + + prefix = data_args.source_prefix if data_args.source_prefix is not None else "" + + # Preprocessing the datasets. + # We need to tokenize inputs and targets. + if training_args.do_train: + column_names = raw_datasets["train"].column_names + elif training_args.do_eval: + column_names = raw_datasets["validation"].column_names + elif training_args.do_predict: + column_names = raw_datasets["test"].column_names + else: + logger.info("There is nothing to do. Please pass `do_train`, `do_eval` and/or `do_predict`.") + return + + # Get the column names for input/target. + prompt_column = data_args.prompt_column + response_column = data_args.response_column + + # Temporarily set max_target_length for training. + max_target_length = data_args.max_target_length + + def preprocess_function_eval(examples): + inputs, targets = [], [] + for i in range(len(examples[prompt_column])): + if examples[prompt_column][i] and examples[response_column][i]: + inputs.append(examples[prompt_column][i]) + targets.append(examples[response_column][i]) + + inputs = [prefix + inp for inp in inputs] + model_inputs = tokenizer(inputs, max_length=data_args.max_source_length, truncation=True) + labels = tokenizer(text_target=targets, max_length=max_target_length, truncation=True) + + if data_args.ignore_pad_token_for_loss: + labels["input_ids"] = [ + [(l if l != tokenizer.pad_token_id else -100) for l in label] for label in labels["input_ids"] + ] + model_inputs["labels"] = labels["input_ids"] + + return model_inputs + + def preprocess_function_train(examples): + max_seq_length = data_args.max_source_length + data_args.max_target_length + + model_inputs = { + "input_ids": [], + "labels": [], + } + for i in range(len(examples[prompt_column])): + if examples[prompt_column][i] and examples[response_column][i]: + prompt, answer = examples[prompt_column][i], examples[response_column][i] + prompt = prefix + prompt + a_ids = tokenizer.encode(text=prompt, add_special_tokens=False) + b_ids = tokenizer.encode(text=answer, add_special_tokens=False) + + if len(a_ids) > data_args.max_source_length - 1: + a_ids = a_ids[: data_args.max_source_length - 1] + + if len(b_ids) > data_args.max_target_length - 2: + b_ids = b_ids[: data_args.max_target_length - 2] + + input_ids = a_ids + [150001, 150004] + b_ids + [150005] + + context_length = input_ids.index(150004) + mask_position = context_length - 1 + labels = [-100] * context_length + input_ids[mask_position+1:] + + pad_len = max_seq_length - len(input_ids) + input_ids = input_ids + [tokenizer.pad_token_id] * pad_len + labels = labels + [tokenizer.pad_token_id] * pad_len + + model_inputs["input_ids"].append(input_ids) + model_inputs["labels"].append(labels) + + return model_inputs + + def print_dataset_example(example): + print("input_ids",example["input_ids"]) + print("inputs", tokenizer.decode(example["input_ids"])) + print("label_ids", example["labels"]) + print("labels", tokenizer.decode(example["labels"])) + + if training_args.do_train: + if "train" not in raw_datasets: + raise ValueError("--do_train requires a train dataset") + train_dataset = raw_datasets["train"] + if data_args.max_train_samples is not None: + max_train_samples = min(len(train_dataset), data_args.max_train_samples) + train_dataset = train_dataset.select(range(max_train_samples)) + with training_args.main_process_first(desc="train dataset map pre-processing"): + train_dataset = train_dataset.map( + preprocess_function_train, + batched=True, + num_proc=data_args.preprocessing_num_workers, + remove_columns=column_names, + load_from_cache_file=not data_args.overwrite_cache, + desc="Running tokenizer on train dataset", + ) + print_dataset_example(train_dataset[0]) + + if training_args.do_eval: + max_target_length = data_args.val_max_target_length + if "validation" not in raw_datasets: + raise ValueError("--do_eval requires a validation dataset") + eval_dataset = raw_datasets["validation"] + if data_args.max_eval_samples is not None: + max_eval_samples = min(len(eval_dataset), data_args.max_eval_samples) + eval_dataset = eval_dataset.select(range(max_eval_samples)) + with training_args.main_process_first(desc="validation dataset map pre-processing"): + eval_dataset = eval_dataset.map( + preprocess_function_eval, + batched=True, + num_proc=data_args.preprocessing_num_workers, + remove_columns=column_names, + load_from_cache_file=not data_args.overwrite_cache, + desc="Running tokenizer on validation dataset", + ) + print_dataset_example(eval_dataset[0]) + + if training_args.do_predict: + max_target_length = data_args.val_max_target_length + if "test" not in raw_datasets: + raise ValueError("--do_predict requires a test dataset") + predict_dataset = raw_datasets["test"] + if data_args.max_predict_samples is not None: + max_predict_samples = min(len(predict_dataset), data_args.max_predict_samples) + predict_dataset = predict_dataset.select(range(max_predict_samples)) + with training_args.main_process_first(desc="prediction dataset map pre-processing"): + predict_dataset = predict_dataset.map( + preprocess_function_eval, + batched=True, + num_proc=data_args.preprocessing_num_workers, + remove_columns=column_names, + load_from_cache_file=not data_args.overwrite_cache, + desc="Running tokenizer on prediction dataset", + ) + print_dataset_example(predict_dataset[0]) + + # Data collator + label_pad_token_id = -100 if data_args.ignore_pad_token_for_loss else tokenizer.pad_token_id + data_collator = DataCollatorForSeq2Seq( + tokenizer, + model=model, + label_pad_token_id=label_pad_token_id, + pad_to_multiple_of=None, + ) + + # Metric + def compute_metrics(eval_preds): + preds, labels = eval_preds + if isinstance(preds, tuple): + preds = preds[0] + decoded_preds = tokenizer.batch_decode(preds, skip_special_tokens=True) + if data_args.ignore_pad_token_for_loss: + # Replace -100 in the labels as we can't decode them. + labels = np.where(labels != -100, labels, tokenizer.pad_token_id) + decoded_labels = tokenizer.batch_decode(labels, skip_special_tokens=True) + + score_dict = { + "rouge-1": [], + "rouge-2": [], + "rouge-l": [], + "bleu-4": [] + } + for pred, label in zip(decoded_preds, decoded_labels): + hypothesis = list(jieba.cut(pred)) + reference = list(jieba.cut(label)) + rouge = Rouge() + scores = rouge.get_scores(' '.join(hypothesis) , ' '.join(reference)) + result = scores[0] + + for k, v in result.items(): + score_dict[k].append(round(v["f"] * 100, 4)) + bleu_score = sentence_bleu([list(label)], list(pred)) + score_dict["bleu-4"].append(round(bleu_score * 100, 4)) + + for k, v in score_dict.items(): + score_dict[k] = float(np.mean(v)) + return score_dict + + # Override the decoding parameters of Seq2SeqTrainer + training_args.generation_max_length = ( + training_args.generation_max_length + if training_args.generation_max_length is not None + else data_args.val_max_target_length + ) + training_args.generation_num_beams = ( + data_args.num_beams if data_args.num_beams is not None else training_args.generation_num_beams + ) + # Initialize our Trainer + trainer = Seq2SeqTrainer( + model=model, + args=training_args, + train_dataset=train_dataset if training_args.do_train else None, + eval_dataset=eval_dataset if training_args.do_eval else None, + tokenizer=tokenizer, + data_collator=data_collator, + compute_metrics=compute_metrics if training_args.predict_with_generate else None, + ) + + # Training + if training_args.do_train: + checkpoint = None + if training_args.resume_from_checkpoint is not None: + checkpoint = training_args.resume_from_checkpoint + # elif last_checkpoint is not None: + # checkpoint = last_checkpoint + model.gradient_checkpointing_enable() + model.enable_input_require_grads() + train_result = trainer.train(resume_from_checkpoint=checkpoint) + # trainer.save_model() # Saves the tokenizer too for easy upload + + metrics = train_result.metrics + max_train_samples = ( + data_args.max_train_samples if data_args.max_train_samples is not None else len(train_dataset) + ) + metrics["train_samples"] = min(max_train_samples, len(train_dataset)) + + trainer.log_metrics("train", metrics) + trainer.save_metrics("train", metrics) + trainer.save_state() + + # Evaluation + results = {} + if training_args.do_eval: + logger.info("*** Evaluate ***") + metrics = trainer.evaluate(metric_key_prefix="eval", do_sample=True, top_p=0.7, max_length=512, temperature=0.95) + max_eval_samples = data_args.max_eval_samples if data_args.max_eval_samples is not None else len(eval_dataset) + metrics["eval_samples"] = min(max_eval_samples, len(eval_dataset)) + + trainer.log_metrics("eval", metrics) + trainer.save_metrics("eval", metrics) + + if training_args.do_predict: + logger.info("*** Predict ***") + + predict_results = trainer.predict(predict_dataset, metric_key_prefix="predict", max_length=512, do_sample=True, top_p=0.7, temperature=0.95) + metrics = predict_results.metrics + max_predict_samples = ( + data_args.max_predict_samples if data_args.max_predict_samples is not None else len(predict_dataset) + ) + metrics["predict_samples"] = min(max_predict_samples, len(predict_dataset)) + + trainer.log_metrics("predict", metrics) + trainer.save_metrics("predict", metrics) + + if trainer.is_world_process_zero(): + if training_args.predict_with_generate: + predictions = tokenizer.batch_decode( + predict_results.predictions, skip_special_tokens=True, clean_up_tokenization_spaces=True + ) + predictions = [pred.strip() for pred in predictions] + labels = tokenizer.batch_decode( + predict_results.label_ids, skip_special_tokens=True, clean_up_tokenization_spaces=True + ) + labels = [label.strip() for label in labels] + output_prediction_file = os.path.join(training_args.output_dir, "generated_predictions.txt") + with open(output_prediction_file, "w") as writer: + for p, l in zip(predictions, labels): + writer.write(json.dumps({"labels": l, "predict": p}, ensure_ascii=False)) + return results + + +def _mp_fn(index): + # For xla_spawn (TPUs) + main() + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/ptuning/train.sh b/ptuning/train.sh new file mode 100644 index 0000000..6988596 --- /dev/null +++ b/ptuning/train.sh @@ -0,0 +1,26 @@ +PRE_SEQ_LEN=8 +LR=1e-2 + +CUDA_VISIBLE_DEVICES=0 python3 main.py \ + --do_train \ + --train_file AdvertiseGen/train.json \ + --validation_file AdvertiseGen/dev.json \ + --prompt_column content \ + --response_column summary \ + --overwrite_cache \ + --model_name_or_path /mnt/vepfs/workspace/zxdu/chatglm_6b \ + --output_dir output/adgen-chatglm-6b-pt-$PRE_SEQ_LEN-$LR-dev \ + --overwrite_output_dir \ + --max_source_length 64 \ + --max_target_length 64 \ + --per_device_train_batch_size 8 \ + --per_device_eval_batch_size 1 \ + --gradient_accumulation_steps 2 \ + --predict_with_generate \ + --max_steps 3000 \ + --logging_steps 10 \ + --save_steps 1000 \ + --learning_rate $LR \ + --pre_seq_len $PRE_SEQ_LEN \ + --quantization_bit 4 + diff --git a/ptuning/trainer_seq2seq.py b/ptuning/trainer_seq2seq.py new file mode 100644 index 0000000..0087786 --- /dev/null +++ b/ptuning/trainer_seq2seq.py @@ -0,0 +1,245 @@ +# Copyright 2020 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Any, Dict, List, Optional, Tuple, Union + +import torch +from torch import nn +from torch.utils.data import Dataset + +from transformers.deepspeed import is_deepspeed_zero3_enabled +from transformers.trainer import Trainer +from transformers.trainer_utils import PredictionOutput +from transformers.utils import logging + + +logger = logging.get_logger(__name__) + + +class Seq2SeqTrainer(Trainer): + def evaluate( + self, + eval_dataset: Optional[Dataset] = None, + ignore_keys: Optional[List[str]] = None, + metric_key_prefix: str = "eval", + **gen_kwargs + ) -> Dict[str, float]: + """ + Run evaluation and returns metrics. + + The calling script will be responsible for providing a method to compute metrics, as they are task-dependent + (pass it to the init `compute_metrics` argument). + + You can also subclass and override this method to inject custom behavior. + + Args: + eval_dataset (`Dataset`, *optional*): + Pass a dataset if you wish to override `self.eval_dataset`. If it is an [`~datasets.Dataset`], columns + not accepted by the `model.forward()` method are automatically removed. It must implement the `__len__` + method. + ignore_keys (`List[str]`, *optional*): + A list of keys in the output of your model (if it is a dictionary) that should be ignored when + gathering predictions. + metric_key_prefix (`str`, *optional*, defaults to `"eval"`): + An optional prefix to be used as the metrics key prefix. For example the metrics "bleu" will be named + "eval_bleu" if the prefix is `"eval"` (default) + max_length (`int`, *optional*): + The maximum target length to use when predicting with the generate method. + num_beams (`int`, *optional*): + Number of beams for beam search that will be used when predicting with the generate method. 1 means no + beam search. + gen_kwargs: + Additional `generate` specific kwargs. + + Returns: + A dictionary containing the evaluation loss and the potential metrics computed from the predictions. The + dictionary also contains the epoch number which comes from the training state. + """ + + gen_kwargs = gen_kwargs.copy() + if gen_kwargs.get("max_length") is None and gen_kwargs.get("max_new_tokens") is None: + gen_kwargs["max_length"] = self.args.generation_max_length + gen_kwargs["num_beams"] = ( + gen_kwargs["num_beams"] if gen_kwargs.get("num_beams") is not None else self.args.generation_num_beams + ) + self._gen_kwargs = gen_kwargs + + return super().evaluate(eval_dataset, ignore_keys=ignore_keys, metric_key_prefix=metric_key_prefix) + + def predict( + self, + test_dataset: Dataset, + ignore_keys: Optional[List[str]] = None, + metric_key_prefix: str = "test", + **gen_kwargs + ) -> PredictionOutput: + """ + Run prediction and returns predictions and potential metrics. + + Depending on the dataset and your use case, your test dataset may contain labels. In that case, this method + will also return metrics, like in `evaluate()`. + + Args: + test_dataset (`Dataset`): + Dataset to run the predictions on. If it is a [`~datasets.Dataset`], columns not accepted by the + `model.forward()` method are automatically removed. Has to implement the method `__len__` + ignore_keys (`List[str]`, *optional*): + A list of keys in the output of your model (if it is a dictionary) that should be ignored when + gathering predictions. + metric_key_prefix (`str`, *optional*, defaults to `"eval"`): + An optional prefix to be used as the metrics key prefix. For example the metrics "bleu" will be named + "eval_bleu" if the prefix is `"eval"` (default) + max_length (`int`, *optional*): + The maximum target length to use when predicting with the generate method. + num_beams (`int`, *optional*): + Number of beams for beam search that will be used when predicting with the generate method. 1 means no + beam search. + gen_kwargs: + Additional `generate` specific kwargs. + + + + If your predictions or labels have different sequence lengths (for instance because you're doing dynamic + padding in a token classification task) the predictions will be padded (on the right) to allow for + concatenation into one array. The padding index is -100. + + + + Returns: *NamedTuple* A namedtuple with the following keys: + + - predictions (`np.ndarray`): The predictions on `test_dataset`. + - label_ids (`np.ndarray`, *optional*): The labels (if the dataset contained some). + - metrics (`Dict[str, float]`, *optional*): The potential dictionary of metrics (if the dataset contained + labels). + """ + + gen_kwargs = gen_kwargs.copy() + if gen_kwargs.get("max_length") is None and gen_kwargs.get("max_new_tokens") is None: + gen_kwargs["max_length"] = self.args.generation_max_length + gen_kwargs["num_beams"] = ( + gen_kwargs["num_beams"] if gen_kwargs.get("num_beams") is not None else self.args.generation_num_beams + ) + self._gen_kwargs = gen_kwargs + + + return super().predict(test_dataset, ignore_keys=ignore_keys, metric_key_prefix=metric_key_prefix) + + def prediction_step( + self, + model: nn.Module, + inputs: Dict[str, Union[torch.Tensor, Any]], + prediction_loss_only: bool, + ignore_keys: Optional[List[str]] = None, + ) -> Tuple[Optional[float], Optional[torch.Tensor], Optional[torch.Tensor]]: + """ + Perform an evaluation step on `model` using `inputs`. + + Subclass and override to inject custom behavior. + + Args: + model (`nn.Module`): + The model to evaluate. + inputs (`Dict[str, Union[torch.Tensor, Any]]`): + The inputs and targets of the model. + + The dictionary will be unpacked before being fed to the model. Most models expect the targets under the + argument `labels`. Check your model's documentation for all accepted arguments. + prediction_loss_only (`bool`): + Whether or not to return the loss only. + + Return: + Tuple[Optional[float], Optional[torch.Tensor], Optional[torch.Tensor]]: A tuple with the loss, logits and + labels (each being optional). + """ + + if not self.args.predict_with_generate or prediction_loss_only: + return super().prediction_step( + model, inputs, prediction_loss_only=prediction_loss_only, ignore_keys=ignore_keys + ) + + has_labels = "labels" in inputs + inputs = self._prepare_inputs(inputs) + + # XXX: adapt synced_gpus for fairscale as well + gen_kwargs = self._gen_kwargs.copy() + if gen_kwargs.get("max_length") is None and gen_kwargs.get("max_new_tokens") is None: + gen_kwargs["max_length"] = self.model.config.max_length + gen_kwargs["num_beams"] = ( + gen_kwargs["num_beams"] if gen_kwargs.get("num_beams") is not None else self.model.config.num_beams + ) + default_synced_gpus = True if is_deepspeed_zero3_enabled() else False + gen_kwargs["synced_gpus"] = ( + gen_kwargs["synced_gpus"] if gen_kwargs.get("synced_gpus") is not None else default_synced_gpus + ) + + if "attention_mask" in inputs: + gen_kwargs["attention_mask"] = inputs.get("attention_mask", None) + if "global_attention_mask" in inputs: + gen_kwargs["global_attention_mask"] = inputs.get("global_attention_mask", None) + + # prepare generation inputs + # some encoder-decoder models can have varying encoder's and thus + # varying model input names + if hasattr(self.model, "encoder") and self.model.encoder.main_input_name != self.model.main_input_name: + generation_inputs = inputs[self.model.encoder.main_input_name] + else: + generation_inputs = inputs[self.model.main_input_name] + + gen_kwargs["input_ids"] = generation_inputs + generated_tokens = self.model.generate(**gen_kwargs) + generated_tokens = generated_tokens[:, generation_inputs.size()[-1]:] + + # in case the batch is shorter than max length, the output should be padded + if gen_kwargs.get("max_length") is not None and generated_tokens.shape[-1] < gen_kwargs["max_length"]: + generated_tokens = self._pad_tensors_to_max_len(generated_tokens, gen_kwargs["max_length"]) + elif gen_kwargs.get("max_new_tokens") is not None and generated_tokens.shape[-1] < ( + gen_kwargs["max_new_tokens"] + 1 + ): + generated_tokens = self._pad_tensors_to_max_len(generated_tokens, gen_kwargs["max_new_tokens"] + 1) + + loss = None + + if self.args.prediction_loss_only: + return (loss, None, None) + + if has_labels: + labels = inputs["labels"] + if gen_kwargs.get("max_length") is not None and labels.shape[-1] < gen_kwargs["max_length"]: + labels = self._pad_tensors_to_max_len(labels, gen_kwargs["max_length"]) + elif gen_kwargs.get("max_new_tokens") is not None and labels.shape[-1] < ( + gen_kwargs["max_new_tokens"] + 1 + ): + labels = self._pad_tensors_to_max_len(labels, (gen_kwargs["max_new_tokens"] + 1)) + else: + labels = None + + return (loss, generated_tokens, labels) + + def _pad_tensors_to_max_len(self, tensor, max_length): + if self.tokenizer is not None and hasattr(self.tokenizer, "pad_token_id"): + # If PAD token is not defined at least EOS token has to be defined + pad_token_id = ( + self.tokenizer.pad_token_id if self.tokenizer.pad_token_id is not None else self.tokenizer.eos_token_id + ) + else: + if self.model.config.pad_token_id is not None: + pad_token_id = self.model.config.pad_token_id + else: + raise ValueError("Pad_token_id must be set in the configuration of the model, in order to pad tensors") + + padded_tensor = pad_token_id * torch.ones( + (tensor.shape[0], max_length), dtype=tensor.dtype, device=tensor.device + ) + padded_tensor[:, : tensor.shape[-1]] = tensor + return padded_tensor From a100769153b8718288d8825c28e003e1af87077c Mon Sep 17 00:00:00 2001 From: duzx16 Date: Fri, 31 Mar 2023 10:46:44 +0800 Subject: [PATCH 29/54] Add P-Tuning v2 --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 3d4e23b..6573e8f 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ ## 介绍 ChatGLM-6B 是一个开源的、支持中英双语的对话语言模型，基于 [General Language Model (GLM)](https://github.com/THUDM/GLM) 架构，具有 62 亿参数。结合模型量化技术，用户可以在消费级的显卡上进行本地部署（INT4 量化级别下最低只需 6GB 显存）。 -ChatGLM-6B 使用了和 ChatGPT 相似的技术，针对中文问答和对话进行了优化。经过约 1T 标识符的中英双语训练，辅以监督微调、反馈自助、人类反馈强化学习等技术的加持，62 亿参数的 ChatGLM-6B 已经能生成相当符合人类偏好的回答。更多信息请参考我们的[博客](https://chatglm.cn/blog)。 +ChatGLM-6B 使用了和 ChatGPT 相似的技术，针对中文问答和对话进行了优化。经过约 1T 标识符的中英双语训练，辅以监督微调、反馈自助、人类反馈强化学习等技术的加持，62 亿参数的 ChatGLM-6B 已经能生成相当符合人类偏好的回答。同时实现了基于P-Tuning v2的[模型微调](ptuning/README.md)（INT4量化级别下最低只需 8GB 显存）。更多信息请参考我们的[博客](https://chatglm.cn/blog)。 不过，由于 ChatGLM-6B 的规模较小，目前已知其具有相当多的[**局限性**](#局限性)，如事实性/数学逻辑错误，可能生成有害/有偏见内容，较弱的上下文能力，自我认知混乱，以及对英文指示生成与中文指示完全矛盾的内容。请大家在使用前了解这些问题，以免产生误解。更大的基于1300亿参数 [GLM-130B](https://github.com/THUDM/GLM-130B) 的 ChatGLM 正在内测开发中。 From 77da04683969f3a76a03c8d92ae7ebc76732f864 Mon Sep 17 00:00:00 2001 From: duzx16 Date: Fri, 31 Mar 2023 10:49:21 +0800 Subject: [PATCH 30/54] Update model path --- ptuning/train.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ptuning/train.sh b/ptuning/train.sh index 6988596..7f9ff6f 100644 --- a/ptuning/train.sh +++ b/ptuning/train.sh @@ -8,7 +8,7 @@ CUDA_VISIBLE_DEVICES=0 python3 main.py \ --prompt_column content \ --response_column summary \ --overwrite_cache \ - --model_name_or_path /mnt/vepfs/workspace/zxdu/chatglm_6b \ + --model_name_or_path THUDM/chatglm-6b \ --output_dir output/adgen-chatglm-6b-pt-$PRE_SEQ_LEN-$LR-dev \ --overwrite_output_dir \ --max_source_length 64 \ From 971a6fbb20d650563e5096bc22351c08e6fabbed Mon Sep 17 00:00:00 2001 From: duzx16 Date: Fri, 31 Mar 2023 11:27:29 +0800 Subject: [PATCH 31/54] Updaet ADGEN link --- ptuning/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ptuning/README.md b/ptuning/README.md index 1fb0ea6..f3f365a 100644 --- a/ptuning/README.md +++ b/ptuning/README.md @@ -20,7 +20,7 @@ ADGEN 数据集任务为根据输入（content）生成一段广告词（summary } ``` -从 [Google Drive](https://drive.google.com/file/d/13_vf0xRTQsyneRKdD1bZIr93vBGOczrk/view?usp=sharing) 或者 [Tsinghua Cloud]() 下载处理好的 ADGEN数据集，将解压后的 `AdvertiseGen` 目录放到本目录下。 +从 [Google Drive](https://drive.google.com/file/d/13_vf0xRTQsyneRKdD1bZIr93vBGOczrk/view?usp=sharing) 或者 [Tsinghua Cloud](https://cloud.tsinghua.edu.cn/f/b3f119a008264b1cabd1/?dl=1) 下载处理好的 ADGEN 数据集，将解压后的 `AdvertiseGen` 目录放到本目录下。 ### 训练 运行以下指令进行训练： From d2645d881653d5605a8e20a1d4725188493d8c3b Mon Sep 17 00:00:00 2001 From: duzx16 Date: Fri, 31 Mar 2023 11:28:13 +0800 Subject: [PATCH 32/54] Update batch size --- ptuning/train.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ptuning/train.sh b/ptuning/train.sh index 7f9ff6f..c0ffce1 100644 --- a/ptuning/train.sh +++ b/ptuning/train.sh @@ -8,14 +8,14 @@ CUDA_VISIBLE_DEVICES=0 python3 main.py \ --prompt_column content \ --response_column summary \ --overwrite_cache \ - --model_name_or_path THUDM/chatglm-6b \ + --model_name_or_path /mnt/vepfs/workspace/zxdu/chatglm_6b \ --output_dir output/adgen-chatglm-6b-pt-$PRE_SEQ_LEN-$LR-dev \ --overwrite_output_dir \ --max_source_length 64 \ --max_target_length 64 \ - --per_device_train_batch_size 8 \ + --per_device_train_batch_size 1 \ --per_device_eval_batch_size 1 \ - --gradient_accumulation_steps 2 \ + --gradient_accumulation_steps 16 \ --predict_with_generate \ --max_steps 3000 \ --logging_steps 10 \ From 5e818065e4b58bfb7be61984207da76c502077f2 Mon Sep 17 00:00:00 2001 From: duzx16 Date: Fri, 31 Mar 2023 11:29:34 +0800 Subject: [PATCH 33/54] Update memory requirement --- README.md | 2 +- ptuning/README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 6573e8f..2c73467 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ ## 介绍 ChatGLM-6B 是一个开源的、支持中英双语的对话语言模型，基于 [General Language Model (GLM)](https://github.com/THUDM/GLM) 架构，具有 62 亿参数。结合模型量化技术，用户可以在消费级的显卡上进行本地部署（INT4 量化级别下最低只需 6GB 显存）。 -ChatGLM-6B 使用了和 ChatGPT 相似的技术，针对中文问答和对话进行了优化。经过约 1T 标识符的中英双语训练，辅以监督微调、反馈自助、人类反馈强化学习等技术的加持，62 亿参数的 ChatGLM-6B 已经能生成相当符合人类偏好的回答。同时实现了基于P-Tuning v2的[模型微调](ptuning/README.md)（INT4量化级别下最低只需 8GB 显存）。更多信息请参考我们的[博客](https://chatglm.cn/blog)。 +ChatGLM-6B 使用了和 ChatGPT 相似的技术，针对中文问答和对话进行了优化。经过约 1T 标识符的中英双语训练，辅以监督微调、反馈自助、人类反馈强化学习等技术的加持，62 亿参数的 ChatGLM-6B 已经能生成相当符合人类偏好的回答。同时实现了基于P-Tuning v2的[模型微调](ptuning/README.md)（INT4量化级别下最低只需 7GB 显存）。更多信息请参考我们的[博客](https://chatglm.cn/blog)。 不过，由于 ChatGLM-6B 的规模较小，目前已知其具有相当多的[**局限性**](#局限性)，如事实性/数学逻辑错误，可能生成有害/有偏见内容，较弱的上下文能力，自我认知混乱，以及对英文指示生成与中文指示完全矛盾的内容。请大家在使用前了解这些问题，以免产生误解。更大的基于1300亿参数 [GLM-130B](https://github.com/THUDM/GLM-130B) 的 ChatGLM 正在内测开发中。 diff --git a/ptuning/README.md b/ptuning/README.md index f3f365a..a4097f6 100644 --- a/ptuning/README.md +++ b/ptuning/README.md @@ -1,5 +1,5 @@ # ChatGLM-6B-PT -本仓库实现了对于 ChatGLM-6B 模型基于 [P-Tuning v2](https://github.com/THUDM/P-tuning-v2) 的微调。P-Tuning v2将需要微调的参数量减少到原来的0.1%，再通过模型量化、Gradient Checkpoint等方法，最低只需要 8GB 显存即可运行。 +本仓库实现了对于 ChatGLM-6B 模型基于 [P-Tuning v2](https://github.com/THUDM/P-tuning-v2) 的微调。P-Tuning v2将需要微调的参数量减少到原来的0.1%，再通过模型量化、Gradient Checkpoint等方法，最低只需要 7GB 显存即可运行。 下面以 [ADGEN](https://aclanthology.org/D19-1321.pdf) (广告生成) 数据集为例介绍代码的使用方法。 From 24e24d5d6c4f015b3d946c69518dcff10d0f758c Mon Sep 17 00:00:00 2001 From: duzx16 Date: Fri, 31 Mar 2023 11:30:36 +0800 Subject: [PATCH 34/54] Fix model path --- ptuning/train.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ptuning/train.sh b/ptuning/train.sh index c0ffce1..1d03a25 100644 --- a/ptuning/train.sh +++ b/ptuning/train.sh @@ -8,7 +8,7 @@ CUDA_VISIBLE_DEVICES=0 python3 main.py \ --prompt_column content \ --response_column summary \ --overwrite_cache \ - --model_name_or_path /mnt/vepfs/workspace/zxdu/chatglm_6b \ + --model_name_or_path THUDM/chatglm-6b \ --output_dir output/adgen-chatglm-6b-pt-$PRE_SEQ_LEN-$LR-dev \ --overwrite_output_dir \ --max_source_length 64 \ From 99875468dd387b78fb765bb3aa335c5fb7435067 Mon Sep 17 00:00:00 2001 From: Aohan Zeng Date: Fri, 31 Mar 2023 11:46:21 +0800 Subject: [PATCH 35/54] Update README.md --- ptuning/README.md | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/ptuning/README.md b/ptuning/README.md index a4097f6..14011f0 100644 --- a/ptuning/README.md +++ b/ptuning/README.md @@ -1,5 +1,5 @@ # ChatGLM-6B-PT -本仓库实现了对于 ChatGLM-6B 模型基于 [P-Tuning v2](https://github.com/THUDM/P-tuning-v2) 的微调。P-Tuning v2将需要微调的参数量减少到原来的0.1%，再通过模型量化、Gradient Checkpoint等方法，最低只需要 7GB 显存即可运行。 +本仓库实现了对于 ChatGLM-6B 模型基于 [P-Tuning v2](https://github.com/THUDM/P-tuning-v2) 的微调。P-Tuning v2 将需要微调的参数量减少到原来的 0.1%，再通过模型量化、Gradient Checkpoint 等方法，最低只需要 7GB 显存即可运行。 下面以 [ADGEN](https://aclanthology.org/D19-1321.pdf) (广告生成) 数据集为例介绍代码的使用方法。 @@ -29,9 +29,11 @@ bash train.sh ``` `train.sh` 中的`PRE_SEQ_LEN`和 `LR` 分别是 soft prompt 长度和训练的学习率，可以进行调节以取得最佳的效果。 +在默认配置 `per_device_train_batch_size=1`, `gradient_accumulation_steps=16` 下，一次训练迭代会以 1 的批处理大小进行 16 次累加的前后向传播，等效为 16 的总批处理大小，此时最低只需 6.7G 显存。若想在同等批处理大小下提升训练效率，可在二者乘积不变的情况下，加大 `per_device_train_batch_size` 的值，但也会带来更多的显存消耗，请根据实际情况酌情调整。 + ### 推理 -将`evaluate.sh`中的`CHECKPOINT`更改为训练时保存的checkpoint名称，运行以下指令进行模型推理和评测： +将 `evaluate.sh` 中的 `CHECKPOINT` 更改为训练时保存的 checkpoint 名称，运行以下指令进行模型推理和评测： ```shell bash evaluate.sh ``` @@ -43,19 +45,19 @@ bash evaluate.sh #### 示例1 * Input: 类型#上衣\*材质#牛仔布\*颜色#白色\*风格#简约\*图案#刺绣\*衣样式#外套\*衣款式#破洞 * Label: 简约而不简单的牛仔外套,白色的衣身十分百搭。衣身多处有做旧破洞设计,打破单调乏味,增加一丝造型看点。衣身后背处有趣味刺绣装饰,丰富层次感,彰显别样时尚。 -* 微调前Output: 这件上衣的材质是牛仔布,颜色是白色,风格是简约,图案是刺绣,衣样式是外套,衣款式是破洞。 -* 微调后Output: 这是一款简约的牛仔外套,破洞设计,将牛仔布破洞,带来一种随意与个性。破洞的牛仔外套,展现出时尚气息,带来一种休闲感。同时,刺绣图案,让整件外套更加立体。 +* Output[微调前]: 这件上衣的材质是牛仔布,颜色是白色,风格是简约,图案是刺绣,衣样式是外套,衣款式是破洞。 +* Output[微调后]: 这是一款简约的牛仔外套,破洞设计,将牛仔布破洞,带来一种随意与个性。破洞的牛仔外套,展现出时尚气息,带来一种休闲感。同时,刺绣图案,让整件外套更加立体。 #### 示例2 * Input: 类型#裙\*版型#显瘦\*风格#文艺\*风格#简约\*图案#印花\*图案#撞色\*裙下摆#压褶\*裙长#连衣裙\*裙领型#圆领 * Label: 文艺个性的印花连衣裙,藏青色底蕴,低调又大气,撞色太阳花分布整个裙身,绚丽而美好,带来时尚减龄的气质。基础款的舒适圆领,简约不失大方,勾勒精致脸庞。领后是一粒包布扣固定,穿脱十分方便。前片立体的打褶设计,搭配后片压褶的做工,增添层次和空间感,显瘦又有型。 -* 微调前Output: 类型#裙*版型#显瘦*风格#文艺*风格#简约*图案#印花*图案#撞色*裙下摆#压褶*裙长#连衣裙*裙领型#圆领 1\. 连衣裙:简约风格,裙长为膝盖以上,裙领型为圆领。2\. 裙下摆:压褶设计,使裙摆呈现出流畅的褶皱效果。3\. 裙领型:裙领型为圆领,使穿上连衣裙后更加有型。4\. 版型:采用显瘦设计,让连衣裙看起来更加苗条。5\. 风格:文艺风格,让连衣裙更加有内涵和品味。6\. 图案:印花设计,在连衣裙上印有独特的图案。7\. 撞色:采用撞色设计,让连衣裙在色彩上更加鲜明、富有层次感。 -* 微调后Output: 这是一款文艺范的连衣裙,以印花为元素,采用简约的印花,既能够突出文艺气质,又能够展现简约风。在印花的同时又有领子和裙摆的压褶设计,更加凸显文艺气质。简约而不会过于单调,搭配出街,穿着十分舒适。 +* Output[微调前]: 类型#裙*版型#显瘦*风格#文艺*风格#简约*图案#印花*图案#撞色*裙下摆#压褶*裙长#连衣裙*裙领型#圆领 1\. 连衣裙:简约风格,裙长为膝盖以上,裙领型为圆领。2\. 裙下摆:压褶设计,使裙摆呈现出流畅的褶皱效果。3\. 裙领型:裙领型为圆领,使穿上连衣裙后更加有型。4\. 版型:采用显瘦设计,让连衣裙看起来更加苗条。5\. 风格:文艺风格,让连衣裙更加有内涵和品味。6\. 图案:印花设计,在连衣裙上印有独特的图案。7\. 撞色:采用撞色设计,让连衣裙在色彩上更加鲜明、富有层次感。 +* Output[微调后]: 这是一款文艺范的连衣裙,以印花为元素,采用简约的印花,既能够突出文艺气质,又能够展现简约风。在印花的同时又有领子和裙摆的压褶设计,更加凸显文艺气质。简约而不会过于单调,搭配出街,穿着十分舒适。 ## 使用自己的数据集 -修改 `train.sh` 和 `evaluate.sh` 中的 `train_file`、`validation_file`和`test_file`为你自己的json格式数据集路径，并将`prompt_column`和`response_column`改为json文件中输入文本和输出文本对应的key。 +修改 `train.sh` 和 `evaluate.sh` 中的 `train_file`、`validation_file`和`test_file`为你自己的 JSON 格式数据集路径，并将 `prompt_column` 和 `response_column` 改为 JSON 文件中输入文本和输出文本对应的 KEY。 ## 引用 From 4c923c4ed769ee0c1bcd6035c49164412c604cac Mon Sep 17 00:00:00 2001 From: Aohan Zeng Date: Fri, 31 Mar 2023 12:21:09 +0800 Subject: [PATCH 36/54] Update p-tuning-v2 --- README.md | 31 +++++++++++++++---------------- 1 file changed, 15 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index 2c73467..33e6c29 100644 --- a/README.md +++ b/README.md @@ -7,16 +7,16 @@ ## 介绍 ChatGLM-6B 是一个开源的、支持中英双语的对话语言模型，基于 [General Language Model (GLM)](https://github.com/THUDM/GLM) 架构，具有 62 亿参数。结合模型量化技术，用户可以在消费级的显卡上进行本地部署（INT4 量化级别下最低只需 6GB 显存）。 -ChatGLM-6B 使用了和 ChatGPT 相似的技术，针对中文问答和对话进行了优化。经过约 1T 标识符的中英双语训练，辅以监督微调、反馈自助、人类反馈强化学习等技术的加持，62 亿参数的 ChatGLM-6B 已经能生成相当符合人类偏好的回答。同时实现了基于P-Tuning v2的[模型微调](ptuning/README.md)（INT4量化级别下最低只需 7GB 显存）。更多信息请参考我们的[博客](https://chatglm.cn/blog)。 +ChatGLM-6B 使用了和 ChatGPT 相似的技术，针对中文问答和对话进行了优化。经过约 1T 标识符的中英双语训练，辅以监督微调、反馈自助、人类反馈强化学习等技术的加持，62 亿参数的 ChatGLM-6B 已经能生成相当符合人类偏好的回答，更多信息请参考我们的[博客](https://chatglm.cn/blog)。此外，为了方便下游开发者针对自己的应用场景定制模型，我们同时实现了基于 P-Tuning v2 的[高效参数微调方法](ptuning/README.md)，INT4 量化级别下最低只需 7GB 显存即可启动微调。 -不过，由于 ChatGLM-6B 的规模较小，目前已知其具有相当多的[**局限性**](#局限性)，如事实性/数学逻辑错误，可能生成有害/有偏见内容，较弱的上下文能力，自我认知混乱，以及对英文指示生成与中文指示完全矛盾的内容。请大家在使用前了解这些问题，以免产生误解。更大的基于1300亿参数 [GLM-130B](https://github.com/THUDM/GLM-130B) 的 ChatGLM 正在内测开发中。 +不过，由于 ChatGLM-6B 的规模较小，目前已知其具有相当多的[**局限性**](#局限性)，如事实性/数学逻辑错误，可能生成有害/有偏见内容，较弱的上下文能力，自我认知混乱，以及对英文指示生成与中文指示完全矛盾的内容。请大家在使用前了解这些问题，以免产生误解。更大的基于 1300 亿参数 [GLM-130B](https://github.com/THUDM/GLM-130B) 的 ChatGLM 正在内测开发中。 *Read this in [English](README_en.md).* ## 更新信息 -**[2023/03/31]** 增加基于 P-Tuning-v2 的微调实现，最低只需 8GB 显存即可进行模型微调。详见[模型微调](ptuning/README.md)。 +**[2023/03/31]** 增加基于 P-Tuning-v2 的高效参数微调实现，INT4 量化级别下最低只需 7GB 显存即可进行模型微调。详见[高效参数微调方法](ptuning/README.md)。 -**[2023/03/23]** 增加API部署（感谢 [@LemonQu-GIT](https://github.com/LemonQu-GIT)）。增加Embedding量化模型[ChatGLM-6B-INT4-QE](https://huggingface.co/THUDM/chatglm-6b-int4-qe)。增加对基于Apple Silicon的Mac上GPU加速的支持。 +**[2023/03/23]** 增加 API 部署（感谢 [@LemonQu-GIT](https://github.com/LemonQu-GIT)）。增加 Embedding 量化模型 [ChatGLM-6B-INT4-QE](https://huggingface.co/THUDM/chatglm-6b-int4-qe)。增加配备 Apple Silicon 芯片的 Mac 上 GPU 加速的支持。 **[2023/03/19]** 增加流式输出接口 `stream_chat`，已更新到网页版和命令行 Demo。修复输出中的中文标点。增加量化后的模型 [ChatGLM-6B-INT4](https://huggingface.co/THUDM/chatglm-6b-int4) @@ -24,12 +24,11 @@ ChatGLM-6B 使用了和 ChatGPT 相似的技术，针对中文问答和对话进 ### 硬件需求 -| **量化等级** | **最低 GPU 显存** | -| -------------- | ----------------- | -| FP16（无量化） | 13 GB | -| INT8 | 10 GB | -| INT4 | 6 GB | - +| **量化等级** | **最低 GPU 显存**（推理） | **最低 GPU 显存**（高效参数微调） | +| -------------- | ------------------------- | --------------------------------- | +| FP16（无量化） | 13 GB | 14 GB | +| INT8 | 10 GB | 11 GB | +| INT4 | 6 GB | 7 GB | ### 环境安装 使用 pip 安装依赖：`pip install -r requirements.txt`，其中 `transformers` 库版本推荐为 `4.26.1`，但理论上不低于 `4.23.1` 即可。 @@ -59,7 +58,7 @@ ChatGLM-6B 使用了和 ChatGPT 相似的技术，针对中文问答和对话进 如果这些方法无法帮助你入睡,你可以考虑咨询医生或睡眠专家,寻求进一步的建议。 ``` -完整的模型实现可以在 [Hugging Face Hub](https://huggingface.co/THUDM/chatglm-6b) 上查看。如果你从 Hugging Face Hub 上下载checkpoint的速度较慢，也可以从[这里](https://cloud.tsinghua.edu.cn/d/fb9f16d6dc8f482596c2/)手动下载。 +完整的模型实现可以在 [Hugging Face Hub](https://huggingface.co/THUDM/chatglm-6b) 上查看。如果你从 Hugging Face Hub 上下载 checkpoint 的速度较慢，也可以从[这里](https://cloud.tsinghua.edu.cn/d/fb9f16d6dc8f482596c2/)手动下载。 ### Demo @@ -94,14 +93,14 @@ python web_demo.py python cli_demo.py ``` -程序会在命令行中进行交互式的对话，在命令行中输入指示并回车即可生成回复，输入`clear`可以清空对话历史，输入`stop`终止程序。 +程序会在命令行中进行交互式的对话，在命令行中输入指示并回车即可生成回复，输入 `clear` 可以清空对话历史，输入 `stop` 终止程序。 ### API部署 -首先需要安装额外的依赖`pip install fastapi uvicorn`，然后运行仓库中的[api.py](api.py)： +首先需要安装额外的依赖 `pip install fastapi uvicorn`，然后运行仓库中的 [api.py](api.py)： ```shell python api.py ``` -默认部署在本地的8000端口，通过POST方法进行调用 +默认部署在本地的 8000 端口，通过 POST 方法进行调用 ```shell curl -X POST "http://127.0.0.1:8000" \ -H 'Content-Type: application/json' \ @@ -167,7 +166,7 @@ model = AutoModel.from_pretrained("your local path", trust_remote_code=True).hal ``` 即可使用在 Mac 上使用 GPU 加速模型推理。 -## 模型微调 +## 高效参数微调 详见 [ptuning/README.md](ptuning/README.md)。 ## ChatGLM-6B 示例 @@ -266,7 +265,7 @@ model = AutoModel.from_pretrained("your local path", trust_remote_code=True).hal 以下是部分针对本项目的教程/文档： * [Windows部署文档](https://github.com/ZhangErling/ChatGLM-6B/blob/main/deployment_windows.md) -如果你有其他好的项目/教程的话，欢迎参照上述格式添加到README中并提出 [PR](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/creating-a-pull-request-from-a-fork). +如果你有其他好的项目/教程的话，欢迎参照上述格式添加到 README 中并提出 [Pull Request](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/creating-a-pull-request-from-a-fork)。 ## 引用 From 9853cd2c9777b2e2289dc65371c430e918ff0118 Mon Sep 17 00:00:00 2001 From: Aohan Zeng Date: Fri, 31 Mar 2023 12:26:09 +0800 Subject: [PATCH 37/54] Update README.md --- ptuning/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ptuning/README.md b/ptuning/README.md index 14011f0..a9b7f41 100644 --- a/ptuning/README.md +++ b/ptuning/README.md @@ -27,9 +27,9 @@ ADGEN 数据集任务为根据输入（content）生成一段广告词（summary ```shell bash train.sh ``` -`train.sh` 中的`PRE_SEQ_LEN`和 `LR` 分别是 soft prompt 长度和训练的学习率，可以进行调节以取得最佳的效果。 +`train.sh` 中的 `PRE_SEQ_LEN` 和 `LR` 分别是 soft prompt 长度和训练的学习率，可以进行调节以取得最佳的效果。P-Tuning-v2 方法会冻结全部的模型参数，可通过调整 `quantization_bit` 来被原始模型的量化等级，不加此选项则为 FP16 精度加载。 -在默认配置 `per_device_train_batch_size=1`, `gradient_accumulation_steps=16` 下，一次训练迭代会以 1 的批处理大小进行 16 次累加的前后向传播，等效为 16 的总批处理大小，此时最低只需 6.7G 显存。若想在同等批处理大小下提升训练效率，可在二者乘积不变的情况下，加大 `per_device_train_batch_size` 的值，但也会带来更多的显存消耗，请根据实际情况酌情调整。 +在默认配置 `quantization_bit=4`、`per_device_train_batch_size=1`、`gradient_accumulation_steps=16` 下，INT4 的模型参数被冻结，一次训练迭代会以 1 的批处理大小进行 16 次累加的前后向传播，等效为 16 的总批处理大小，此时最低只需 6.7G 显存。若想在同等批处理大小下提升训练效率，可在二者乘积不变的情况下，加大 `per_device_train_batch_size` 的值，但也会带来更多的显存消耗，请根据实际情况酌情调整。 ### 推理 From ba93cafa7d828966a2ca4c86b229afeea3381ab8 Mon Sep 17 00:00:00 2001 From: Aohan Zeng Date: Fri, 31 Mar 2023 12:29:39 +0800 Subject: [PATCH 38/54] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 33e6c29..60ee417 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,7 @@ ChatGLM-6B 使用了和 ChatGPT 相似的技术，针对中文问答和对话进 | **量化等级** | **最低 GPU 显存**（推理） | **最低 GPU 显存**（高效参数微调） | | -------------- | ------------------------- | --------------------------------- | | FP16（无量化） | 13 GB | 14 GB | -| INT8 | 10 GB | 11 GB | +| INT8 | 8 GB | 9 GB | | INT4 | 6 GB | 7 GB | ### 环境安装 From 7e84262ddc342798a04ad034fee75ad3432d75e3 Mon Sep 17 00:00:00 2001 From: Shaw Date: Fri, 31 Mar 2023 14:13:51 +0800 Subject: [PATCH 39/54] Update README.md --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 60ee417..622a316 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,7 @@ ChatGLM-6B 使用了和 ChatGPT 相似的技术，针对中文问答和对话进 *Read this in [English](README_en.md).* ## 更新信息 -**[2023/03/31]** 增加基于 P-Tuning-v2 的高效参数微调实现，INT4 量化级别下最低只需 7GB 显存即可进行模型微调。详见[高效参数微调方法](ptuning/README.md)。 +**[2023/03/31]** 增加基于 [P-Tuning-v2](https://github.com/THUDM/P-tuning-v2) 的高效参数微调实现，INT4 量化级别下最低只需 7GB 显存即可进行模型微调。详见[高效参数微调方法](ptuning/README.md)。 **[2023/03/23]** 增加 API 部署（感谢 [@LemonQu-GIT](https://github.com/LemonQu-GIT)）。增加 Embedding 量化模型 [ChatGLM-6B-INT4-QE](https://huggingface.co/THUDM/chatglm-6b-int4-qe)。增加配备 Apple Silicon 芯片的 Mac 上 GPU 加速的支持。 @@ -167,7 +167,7 @@ model = AutoModel.from_pretrained("your local path", trust_remote_code=True).hal 即可使用在 Mac 上使用 GPU 加速模型推理。 ## 高效参数微调 -详见 [ptuning/README.md](ptuning/README.md)。 +基于 [P-tuning v2](https://github.com/THUDM/P-tuning-v2) 的高效参数微调。具体使用方法详见 [ptuning/README.md](ptuning/README.md)。 ## ChatGLM-6B 示例 From fdc2c7f70d00b7ce6ca098934b697e31fd8be8fe Mon Sep 17 00:00:00 2001 From: Shaw Date: Fri, 31 Mar 2023 14:48:55 +0800 Subject: [PATCH 40/54] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 622a316..6e8b72a 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ ## 介绍 ChatGLM-6B 是一个开源的、支持中英双语的对话语言模型，基于 [General Language Model (GLM)](https://github.com/THUDM/GLM) 架构，具有 62 亿参数。结合模型量化技术，用户可以在消费级的显卡上进行本地部署（INT4 量化级别下最低只需 6GB 显存）。 -ChatGLM-6B 使用了和 ChatGPT 相似的技术，针对中文问答和对话进行了优化。经过约 1T 标识符的中英双语训练，辅以监督微调、反馈自助、人类反馈强化学习等技术的加持，62 亿参数的 ChatGLM-6B 已经能生成相当符合人类偏好的回答，更多信息请参考我们的[博客](https://chatglm.cn/blog)。此外，为了方便下游开发者针对自己的应用场景定制模型，我们同时实现了基于 P-Tuning v2 的[高效参数微调方法](ptuning/README.md)，INT4 量化级别下最低只需 7GB 显存即可启动微调。 +ChatGLM-6B 使用了和 ChatGPT 相似的技术，针对中文问答和对话进行了优化。经过约 1T 标识符的中英双语训练，辅以监督微调、反馈自助、人类反馈强化学习等技术的加持，62 亿参数的 ChatGLM-6B 已经能生成相当符合人类偏好的回答，更多信息请参考我们的[博客](https://chatglm.cn/blog)。此外，为了方便下游开发者针对自己的应用场景定制模型，我们同时实现了基于 [P-Tuning v2](https://github.com/THUDM/P-tuning-v2) 的高效参数微调方法 [(使用指南)](ptuning/README.md) ，INT4 量化级别下最低只需 7GB 显存即可启动微调。 不过，由于 ChatGLM-6B 的规模较小，目前已知其具有相当多的[**局限性**](#局限性)，如事实性/数学逻辑错误，可能生成有害/有偏见内容，较弱的上下文能力，自我认知混乱，以及对英文指示生成与中文指示完全矛盾的内容。请大家在使用前了解这些问题，以免产生误解。更大的基于 1300 亿参数 [GLM-130B](https://github.com/THUDM/GLM-130B) 的 ChatGLM 正在内测开发中。 From c206e7d9ad2911d1f7fb0294110b7ef361d41a58 Mon Sep 17 00:00:00 2001 From: duzx16 Date: Fri, 31 Mar 2023 15:18:21 +0800 Subject: [PATCH 41/54] Update requirements.txt --- README.md | 2 +- README_en.md | 2 +- ptuning/README.md | 2 +- requirements.txt | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 2c73467..9e072a0 100644 --- a/README.md +++ b/README.md @@ -32,7 +32,7 @@ ChatGLM-6B 使用了和 ChatGPT 相似的技术，针对中文问答和对话进 ### 环境安装 -使用 pip 安装依赖：`pip install -r requirements.txt`，其中 `transformers` 库版本推荐为 `4.26.1`，但理论上不低于 `4.23.1` 即可。 +使用 pip 安装依赖：`pip install -r requirements.txt`，其中 `transformers` 库版本推荐为 `4.27.1`，但理论上不低于 `4.23.1` 即可。 ### 代码调用 diff --git a/README_en.md b/README_en.md index b5b4b62..d5c05bb 100644 --- a/README_en.md +++ b/README_en.md @@ -32,7 +32,7 @@ If you have other good projects, please refer to the above format to add to READ ### Environment Setup -Install the requirements with pip: `pip install -r requirements.txt`. `transformers` library version is recommended to be `4.26.1`, but theoretically any version no lower than `4.23.1` is acceptable. +Install the requirements with pip: `pip install -r requirements.txt`. `transformers` library version is recommended to be `4.27.1`, but theoretically any version no lower than `4.23.1` is acceptable. ### Usage diff --git a/ptuning/README.md b/ptuning/README.md index a4097f6..4434e3a 100644 --- a/ptuning/README.md +++ b/ptuning/README.md @@ -4,7 +4,7 @@ 下面以 [ADGEN](https://aclanthology.org/D19-1321.pdf) (广告生成) 数据集为例介绍代码的使用方法。 ## 软件依赖 -除 ChatGLM-6B 的依赖之外，还需要按照以下依赖 +运行微调需要4.27.1版本的`transformers`。除 ChatGLM-6B 的依赖之外，还需要按照以下依赖 ``` pip install rouge_chinese nltk jieba datasets ``` diff --git a/requirements.txt b/requirements.txt index 2948480..00707fe 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ protobuf>=3.19.5,<3.20.1 -transformers==4.26.1 +transformers==4.27.1 icetk cpm_kernels torch>=1.10 From 08d880141d7450099a579e00877e3318be26da5f Mon Sep 17 00:00:00 2001 From: duzx16 Date: Fri, 31 Mar 2023 16:32:34 +0800 Subject: [PATCH 42/54] Fix revision for loading model --- ptuning/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ptuning/main.py b/ptuning/main.py index d82fccc..5b23234 100644 --- a/ptuning/main.py +++ b/ptuning/main.py @@ -110,7 +110,7 @@ def main(): tokenizer = AutoTokenizer.from_pretrained(model_args.model_name_or_path, trust_remote_code=True) - model = AutoModel.from_pretrained(model_args.model_name_or_path, config=config, revision=True, trust_remote_code=True) + model = AutoModel.from_pretrained(model_args.model_name_or_path, config=config, trust_remote_code=True) model = model.half() if model_args.quantization_bit is not None: From 893706a82d5529e6a99861c31335c99b440f8842 Mon Sep 17 00:00:00 2001 From: rainatam Date: Fri, 31 Mar 2023 18:12:04 +0800 Subject: [PATCH 43/54] Update train script --- ptuning/arguments.py | 4 ++-- ptuning/train.sh | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ptuning/arguments.py b/ptuning/arguments.py index 1c61f97..95d766f 100644 --- a/ptuning/arguments.py +++ b/ptuning/arguments.py @@ -203,8 +203,8 @@ class DataTrainingArguments: def __post_init__(self): - if self.dataset_name is None and self.train_file is None and self.validation_file is None: - raise ValueError("Need either a dataset name or a training/validation file.") + if self.dataset_name is None and self.train_file is None and self.validation_file is None and self.test_file is None: + raise ValueError("Need either a dataset name or a training/validation/test file.") else: if self.train_file is not None: extension = self.train_file.split(".")[-1] diff --git a/ptuning/train.sh b/ptuning/train.sh index 1d03a25..3189829 100644 --- a/ptuning/train.sh +++ b/ptuning/train.sh @@ -9,7 +9,7 @@ CUDA_VISIBLE_DEVICES=0 python3 main.py \ --response_column summary \ --overwrite_cache \ --model_name_or_path THUDM/chatglm-6b \ - --output_dir output/adgen-chatglm-6b-pt-$PRE_SEQ_LEN-$LR-dev \ + --output_dir output/adgen-chatglm-6b-pt-$PRE_SEQ_LEN-$LR \ --overwrite_output_dir \ --max_source_length 64 \ --max_target_length 64 \ From 73f4fe1ffe6af778796a2ed88882af5655062eb0 Mon Sep 17 00:00:00 2001 From: duzx16 Date: Fri, 31 Mar 2023 20:15:35 +0800 Subject: [PATCH 44/54] Add validation file name Use full prediction --- ptuning/evaluate.sh | 2 +- ptuning/train.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ptuning/evaluate.sh b/ptuning/evaluate.sh index db2a8c1..1217ceb 100644 --- a/ptuning/evaluate.sh +++ b/ptuning/evaluate.sh @@ -4,6 +4,7 @@ STEP=3000 CUDA_VISIBLE_DEVICES=0 python3 main.py \ --do_predict \ + --validation_file AdvertiseGen/dev.json \ --test_file AdvertiseGen/dev.json \ --overwrite_cache \ --prompt_column content \ @@ -15,6 +16,5 @@ CUDA_VISIBLE_DEVICES=0 python3 main.py \ --max_target_length 64 \ --per_device_eval_batch_size 1 \ --predict_with_generate \ - --max_predict_samples 10 \ --pre_seq_len $PRE_SEQ_LEN \ --quantization_bit 4 diff --git a/ptuning/train.sh b/ptuning/train.sh index 1d03a25..3189829 100644 --- a/ptuning/train.sh +++ b/ptuning/train.sh @@ -9,7 +9,7 @@ CUDA_VISIBLE_DEVICES=0 python3 main.py \ --response_column summary \ --overwrite_cache \ --model_name_or_path THUDM/chatglm-6b \ - --output_dir output/adgen-chatglm-6b-pt-$PRE_SEQ_LEN-$LR-dev \ + --output_dir output/adgen-chatglm-6b-pt-$PRE_SEQ_LEN-$LR \ --overwrite_output_dir \ --max_source_length 64 \ --max_target_length 64 \ From 7436f0840f16136266e347157a45f7c74f390d15 Mon Sep 17 00:00:00 2001 From: duzx16 Date: Fri, 31 Mar 2023 22:55:36 +0800 Subject: [PATCH 45/54] Add todo --- ptuning/README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ptuning/README.md b/ptuning/README.md index dddae50..6ce6384 100644 --- a/ptuning/README.md +++ b/ptuning/README.md @@ -59,6 +59,10 @@ bash evaluate.sh ## 使用自己的数据集 修改 `train.sh` 和 `evaluate.sh` 中的 `train_file`、`validation_file`和`test_file`为你自己的 JSON 格式数据集路径，并将 `prompt_column` 和 `response_column` 改为 JSON 文件中输入文本和输出文本对应的 KEY。 +## TODO +* [ ] Support for chat data +* [ ] Support for full finetuning + ## 引用 ``` From 7a67ddd61f65e4326c96c7c02ee48027151abccb Mon Sep 17 00:00:00 2001 From: maybeluo Date: Sat, 1 Apr 2023 00:34:19 +0800 Subject: [PATCH 46/54] write generated result with utf-8 --- ptuning/main.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/ptuning/main.py b/ptuning/main.py index 5b23234..d1c4977 100644 --- a/ptuning/main.py +++ b/ptuning/main.py @@ -374,9 +374,10 @@ def main(): ) labels = [label.strip() for label in labels] output_prediction_file = os.path.join(training_args.output_dir, "generated_predictions.txt") - with open(output_prediction_file, "w") as writer: + with open(output_prediction_file, "w", encoding="utf-8") as writer: for p, l in zip(predictions, labels): - writer.write(json.dumps({"labels": l, "predict": p}, ensure_ascii=False)) + res = json.dumps({"labels": l, "predict": p}, ensure_ascii=False) + writer.write(f"{res}\n") return results @@ -386,4 +387,4 @@ def _mp_fn(index): if __name__ == "__main__": - main() \ No newline at end of file + main() From acd4adcb651211bcc4b01c86e92ed09de2edf413 Mon Sep 17 00:00:00 2001 From: Yam Date: Sat, 1 Apr 2023 11:17:35 +0800 Subject: [PATCH 47/54] Update README.md --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index 73ea77f..1f82c03 100644 --- a/README.md +++ b/README.md @@ -169,6 +169,8 @@ model = AutoModel.from_pretrained("your local path", trust_remote_code=True).hal ## 高效参数微调 基于 [P-tuning v2](https://github.com/THUDM/P-tuning-v2) 的高效参数微调。具体使用方法详见 [ptuning/README.md](ptuning/README.md)。 + + ## ChatGLM-6B 示例 以下是一些使用 `web_demo.py` 得到的示例截图。更多 ChatGLM-6B 的可能，等待你来探索发现！ @@ -261,6 +263,7 @@ model = AutoModel.from_pretrained("your local path", trust_remote_code=True).hal 以下是部分基于本仓库开发的开源项目： * [ChatGLM-MNN](https://github.com/wangzhaode/ChatGLM-MNN): 一个基于 MNN 的 ChatGLM-6B C++ 推理实现，支持根据显存大小自动分配计算任务给 GPU 和 CPU * [ChatGLM-Tuning](https://github.com/mymusise/ChatGLM-Tuning): 基于 LoRA 对 ChatGLM-6B 进行微调 +* [Humanable ChatGLM/GPT Fine-tuning | ChatGLM 微调](https://github.com/hscspring/hcgf)：基于 LoRA 进行微调 以下是部分针对本项目的教程/文档： * [Windows部署文档](https://github.com/ZhangErling/ChatGLM-6B/blob/main/deployment_windows.md) From 6dd6f7c7b53604fb8eee2aa27509619752bf3133 Mon Sep 17 00:00:00 2001 From: Ashing Zheng Date: Sat, 1 Apr 2023 13:18:28 +0800 Subject: [PATCH 48/54] chore(cli_demo): strip the query input for `stop` or `clear` command Remove the query space to determine whether it is a stop command or a clear command --- cli_demo.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cli_demo.py b/cli_demo.py index 1c3ff2b..da80fff 100644 --- a/cli_demo.py +++ b/cli_demo.py @@ -31,9 +31,9 @@ def main(): print("欢迎使用 ChatGLM-6B 模型，输入内容即可进行对话，clear 清空对话历史，stop 终止程序") while True: query = input("\n用户：") - if query == "stop": + if query.strip() == "stop": break - if query == "clear": + if query.strip() == "clear": history = [] os.system(clear_command) print("欢迎使用 ChatGLM-6B 模型，输入内容即可进行对话，clear 清空对话历史，stop 终止程序") From e8aba3d3f3e77c6f95211a355f44883b1d213794 Mon Sep 17 00:00:00 2001 From: duzx16 Date: Sat, 1 Apr 2023 23:08:44 +0800 Subject: [PATCH 49/54] Add project links --- README.md | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 1f82c03..9d5386e 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,9 @@ ## 介绍 ChatGLM-6B 是一个开源的、支持中英双语的对话语言模型，基于 [General Language Model (GLM)](https://github.com/THUDM/GLM) 架构，具有 62 亿参数。结合模型量化技术，用户可以在消费级的显卡上进行本地部署（INT4 量化级别下最低只需 6GB 显存）。 -ChatGLM-6B 使用了和 ChatGPT 相似的技术，针对中文问答和对话进行了优化。经过约 1T 标识符的中英双语训练，辅以监督微调、反馈自助、人类反馈强化学习等技术的加持，62 亿参数的 ChatGLM-6B 已经能生成相当符合人类偏好的回答，更多信息请参考我们的[博客](https://chatglm.cn/blog)。此外，为了方便下游开发者针对自己的应用场景定制模型，我们同时实现了基于 [P-Tuning v2](https://github.com/THUDM/P-tuning-v2) 的高效参数微调方法 [(使用指南)](ptuning/README.md) ，INT4 量化级别下最低只需 7GB 显存即可启动微调。 +ChatGLM-6B 使用了和 ChatGPT 相似的技术，针对中文问答和对话进行了优化。经过约 1T 标识符的中英双语训练，辅以监督微调、反馈自助、人类反馈强化学习等技术的加持，62 亿参数的 ChatGLM-6B 已经能生成相当符合人类偏好的回答，更多信息请参考我们的[博客](https://chatglm.cn/blog)。 + +为了方便下游开发者针对自己的应用场景定制模型，我们同时实现了基于 [P-Tuning v2](https://github.com/THUDM/P-tuning-v2) 的高效参数微调方法 [(使用指南)](ptuning/README.md) ，INT4 量化级别下最低只需 7GB 显存即可启动微调。 不过，由于 ChatGLM-6B 的规模较小，目前已知其具有相当多的[**局限性**](#局限性)，如事实性/数学逻辑错误，可能生成有害/有偏见内容，较弱的上下文能力，自我认知混乱，以及对英文指示生成与中文指示完全矛盾的内容。请大家在使用前了解这些问题，以免产生误解。更大的基于 1300 亿参数 [GLM-130B](https://github.com/THUDM/GLM-130B) 的 ChatGLM 正在内测开发中。 @@ -20,6 +22,18 @@ ChatGLM-6B 使用了和 ChatGPT 相似的技术，针对中文问答和对话进 **[2023/03/19]** 增加流式输出接口 `stream_chat`，已更新到网页版和命令行 Demo。修复输出中的中文标点。增加量化后的模型 [ChatGLM-6B-INT4](https://huggingface.co/THUDM/chatglm-6b-int4) +## 友情链接 +以下是部分基于本仓库开发的开源项目： +* [ChatGLM-MNN](https://github.com/wangzhaode/ChatGLM-MNN): 一个基于 MNN 的 ChatGLM-6B C++ 推理实现，支持根据显存大小自动分配计算任务给 GPU 和 CPU +* [ChatGLM-Tuning](https://github.com/mymusise/ChatGLM-Tuning): 基于 LoRA 对 ChatGLM-6B 进行微调。类似的项目还包括 [Humanable ChatGLM/GPT Fine-tuning | ChatGLM 微调](https://github.com/hscspring/hcgf) +* [langchain-ChatGLM](https://github.com/imClumsyPanda/langchain-ChatGLM)：基于本地知识的 ChatGLM 应用，基于LangChain +* [闻达](https://github.com/l15y/wenda)：大型语言模型调用平台，基于 ChatGLM-6B 实现了类 ChatPDF 功能 + +以下是部分针对本项目的教程/文档： +* [Windows部署文档](https://github.com/ZhangErling/ChatGLM-6B/blob/main/deployment_windows.md) + +如果你有其他好的项目/教程的话，欢迎参照上述格式添加到 README 中并提出 [Pull Request](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/creating-a-pull-request-from-a-fork)。 + ## 使用方式 ### 硬件需求 @@ -259,17 +273,6 @@ model = AutoModel.from_pretrained("your local path", trust_remote_code=True).hal 本仓库的代码依照 [Apache-2.0](LICENSE) 协议开源，ChatGLM-6B 模型的权重的使用则需要遵循 [Model License](MODEL_LICENSE)。 -## 友情链接 -以下是部分基于本仓库开发的开源项目： -* [ChatGLM-MNN](https://github.com/wangzhaode/ChatGLM-MNN): 一个基于 MNN 的 ChatGLM-6B C++ 推理实现，支持根据显存大小自动分配计算任务给 GPU 和 CPU -* [ChatGLM-Tuning](https://github.com/mymusise/ChatGLM-Tuning): 基于 LoRA 对 ChatGLM-6B 进行微调 -* [Humanable ChatGLM/GPT Fine-tuning | ChatGLM 微调](https://github.com/hscspring/hcgf)：基于 LoRA 进行微调 - -以下是部分针对本项目的教程/文档： -* [Windows部署文档](https://github.com/ZhangErling/ChatGLM-6B/blob/main/deployment_windows.md) - -如果你有其他好的项目/教程的话，欢迎参照上述格式添加到 README 中并提出 [Pull Request](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/creating-a-pull-request-from-a-fork)。 - ## 引用 如果你觉得我们的工作有帮助的话，请考虑引用下列论文 From 4371f7a57285d94f6c22827418f7cebe2983c0b5 Mon Sep 17 00:00:00 2001 From: duzx16 Date: Sat, 1 Apr 2023 23:09:26 +0800 Subject: [PATCH 50/54] Add padding for evaluation data --- ptuning/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ptuning/main.py b/ptuning/main.py index d1c4977..1776055 100644 --- a/ptuning/main.py +++ b/ptuning/main.py @@ -147,7 +147,7 @@ def main(): targets.append(examples[response_column][i]) inputs = [prefix + inp for inp in inputs] - model_inputs = tokenizer(inputs, max_length=data_args.max_source_length, truncation=True) + model_inputs = tokenizer(inputs, max_length=data_args.max_source_length, truncation=True, padding=True) labels = tokenizer(text_target=targets, max_length=max_target_length, truncation=True) if data_args.ignore_pad_token_for_loss: From 5f8f6bac7b2ca3212ba913aa4cbd85aa2313f632 Mon Sep 17 00:00:00 2001 From: duzx16 Date: Sun, 2 Apr 2023 00:35:40 +0800 Subject: [PATCH 51/54] Change quantization instruction --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 9d5386e..a286cee 100644 --- a/README.md +++ b/README.md @@ -136,7 +136,7 @@ curl -X POST "http://127.0.0.1:8000" \ ```python # 按需修改，目前只支持 4/8 bit 量化 -model = AutoModel.from_pretrained("THUDM/chatglm-6b", trust_remote_code=True).half().quantize(4).cuda() +model = AutoModel.from_pretrained("THUDM/chatglm-6b", trust_remote_code=True).quantize(4).half().cuda() ``` 进行 2 至 3 轮对话后，8-bit 量化下 GPU 显存占用约为 10GB，4-bit 量化下仅需 6GB 占用。随着对话轮数的增多，对应消耗显存也随之增长，由于采用了相对位置编码，理论上 ChatGLM-6B 支持无限长的 context-length，但总长度超过 2048（训练长度）后性能会逐渐下降。 From ca43864f39eaa6a19493ebd812bff25c44c9100e Mon Sep 17 00:00:00 2001 From: duzx16 Date: Sun, 2 Apr 2023 00:35:40 +0800 Subject: [PATCH 52/54] Change quantization instruction --- README.md | 2 +- ptuning/main.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 9d5386e..a286cee 100644 --- a/README.md +++ b/README.md @@ -136,7 +136,7 @@ curl -X POST "http://127.0.0.1:8000" \ ```python # 按需修改，目前只支持 4/8 bit 量化 -model = AutoModel.from_pretrained("THUDM/chatglm-6b", trust_remote_code=True).half().quantize(4).cuda() +model = AutoModel.from_pretrained("THUDM/chatglm-6b", trust_remote_code=True).quantize(4).half().cuda() ``` 进行 2 至 3 轮对话后，8-bit 量化下 GPU 显存占用约为 10GB，4-bit 量化下仅需 6GB 占用。随着对话轮数的增多，对应消耗显存也随之增长，由于采用了相对位置编码，理论上 ChatGLM-6B 支持无限长的 context-length，但总长度超过 2048（训练长度）后性能会逐渐下降。 diff --git a/ptuning/main.py b/ptuning/main.py index 1776055..020c514 100644 --- a/ptuning/main.py +++ b/ptuning/main.py @@ -112,10 +112,10 @@ def main(): model = AutoModel.from_pretrained(model_args.model_name_or_path, config=config, trust_remote_code=True) - model = model.half() if model_args.quantization_bit is not None: print(f"Quantized to {model_args.quantization_bit} bit") model = model.quantize(model_args.quantization_bit) + model = model.half() model.transformer.prefix_encoder.float() prefix = data_args.source_prefix if data_args.source_prefix is not None else "" From c508f62b7009a5d5c0b390fb0e67f297dbcd014e Mon Sep 17 00:00:00 2001 From: duzx16 Date: Sun, 2 Apr 2023 01:59:07 +0800 Subject: [PATCH 53/54] Fix position_ids in prediction --- ptuning/trainer_seq2seq.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ptuning/trainer_seq2seq.py b/ptuning/trainer_seq2seq.py index 0087786..518daa0 100644 --- a/ptuning/trainer_seq2seq.py +++ b/ptuning/trainer_seq2seq.py @@ -185,6 +185,8 @@ class Seq2SeqTrainer(Trainer): if "attention_mask" in inputs: gen_kwargs["attention_mask"] = inputs.get("attention_mask", None) + if "position_ids" in inputs: + gen_kwargs["position_ids"] = inputs.get("position_ids", None) if "global_attention_mask" in inputs: gen_kwargs["global_attention_mask"] = inputs.get("global_attention_mask", None) From 4227999d4c61f17bacf9d09272168cbd46f5b1f5 Mon Sep 17 00:00:00 2001 From: duzx16 Date: Sun, 2 Apr 2023 02:05:03 +0800 Subject: [PATCH 54/54] No padding in colloator --- ptuning/main.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ptuning/main.py b/ptuning/main.py index 020c514..cd56e43 100644 --- a/ptuning/main.py +++ b/ptuning/main.py @@ -262,6 +262,7 @@ def main(): model=model, label_pad_token_id=label_pad_token_id, pad_to_multiple_of=None, + padding=False ) # Metric