Merge 6ba39b421c into 401bf3a8a7

api.py

@@ -2,6 +2,7 @@ from fastapi import FastAPI, Request
 from transformers import AutoTokenizer, AutoModel
 import uvicorn, json, datetime
 import torch
+import asyncio
 
 DEVICE = "cuda"
 DEVICE_ID = "0"
@@ -29,6 +30,8 @@ async def create_item(request: Request):
     max_length = json_post_list.get('max_length')
     top_p = json_post_list.get('top_p')
     temperature = json_post_list.get('temperature')
+
+    def _sync_chat(history):
         response, history = model.chat(tokenizer,
                                        prompt,
                                        history=history,
@@ -48,6 +51,8 @@ async def create_item(request: Request):
         torch_gc()
         return answer
 
+    return await asyncio.to_thread(_sync_chat, history=history)
+
 
 if __name__ == '__main__':
     tokenizer = AutoTokenizer.from_pretrained("THUDM/chatglm-6b", trust_remote_code=True)