github
/
ChatGLM2-6B
mirror of https://github.com/THUDM/ChatGLM2-6B
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Add the client sample paired with openai_api.py

Browse Source 
The openai_api.py is just the server sample code of openai. So I write
this client sample to show how-to send the request w/ the api of openai
and get the response from the server. And I also try the benchmark to get
the throughput in words/sencond.

Usage:
1. Start the server by run "python openai_api.py".
2. Start the client with this file with jupyter-notebook.

Signed-off-by: Alex He <heye_dev@163.com>
pull/504/head
Alex He 1 year ago
parent 80602dcae1
commit 5a31c2c1d3
1 changed files with 206 additions and 0 deletions
Show Stats Download Patch File Download Diff File

206
try_openai_api_client.ipynb
Unescape View File

@ -0,0 +1,206 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": 79,
"metadata": {},
"outputs": [],
"source": [
"import os\n",
"import openai"
]
},
{
"cell_type": "code",
"execution_count": 80,
"metadata": {},
"outputs": [],
"source": [
"openai.api_base = \"http://localhost:8000/v1\"\n",
"openai.api_key = \"none\""
]
},
{
"cell_type": "code",
"execution_count": 81,
"metadata": {},
"outputs": [],
"source": [
"def get_completion(prompt, model=\"chatglm2-6b-model\"):\n",
" messages = [{\"role\": \"user\", \"content\": prompt}]\n",
" response = openai.ChatCompletion.create(\n",
" model=model,\n",
" messages=messages,\n",
" temperature=0,\n",
" max_tokens=2048\n",
" )\n",
" #return response.choices[0].message[\"content\"]\n",
" return response"
]
},
{
"cell_type": "code",
"execution_count": 82,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"返回内容:\n",
"《三国演义》中的曹操是三国时期最具代表性的历史人物之一,被誉为“三国第一谋士”。他是一个极富谋略、谋断和领导力的领袖,也是一位出色的诗人。\n",
"\n",
"曹操在小说中被描绘成一个极具野心、聪明狡猾、冷酷无情的政治家。他为了实现自己的理想,不惜牺牲他人的利益,甚至为了达到目的,不惜杀妻灭子。这种冷酷无情,虽然有些令人震驚,但也反映了他作为一个政治家的决心和勇气。\n",
"\n",
"曹操在小说中也被描绘成一个出色的军事家。他指挥过许多战役,并制定了许多军事战略和计划,如“三顾茅庐”、“赤壁之战”等,都取得了重大胜利。他的军事才能和领导力,使得他成为了三国中最强大的军队的指挥官之一。\n",
"\n",
"曹操在小说中也被描绘成一个很有文化修养的人。他非常热爱文学,精通诗词,也善于书法。这种文化修养不仅彰显出他的品味和才华,也反映出他对于国家和民族的重视,以及他对于文化传承和发展的贡献。\n",
"\n",
"曹操是一个极具矛盾性和复杂性的历史人物。他的聪明、狡猾、冷酷、果断、文化和军事才能都使他成为了三国时期最杰出的政治家和军事家之一。\n"
]
}
],
"source": [
"response=get_completion(\"三百字评价一下下三国演义中的曹操\")\n",
"\n",
"\n",
"# 返回的内容\n",
"print(f\"返回内容:\")\n",
"print(response.choices[0].message[\"content\"])"
]
},
{
"cell_type": "code",
"execution_count": 83,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"返回字数: 439\n"
]
}
],
"source": [
"# 返回的字数\n",
"print(\"返回字数:\", len(response.choices[0].message[\"content\"]))"
]
},
{
"cell_type": "code",
"execution_count": 84,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"第0次返回的字数: 578\n",
"第1次返回的字数: 401\n",
"第2次返回的字数: 599\n",
"第3次返回的字数: 524\n",
"第4次返回的字数: 484\n",
"第5次返回的字数: 761\n",
"第6次返回的字数: 625\n",
"第7次返回的字数: 342\n",
"第8次返回的字数: 515\n",
"第9次返回的字数: 507\n",
"平均每次返回的字数: 533.6\n",
"平均每秒输出的字数：57.0\n"
]
}
],
"source": [
"# Throughput Benchmark\n",
"# 我的测试中，RTX3090的吞吐为每秒约57字\n",
"\n",
"print(\"Throughput Benchmark:\", )\n",
"\n",
"import time\n",
"\n",
"total_words = 0\n",
"start_time = time.time()\n",
"for i in range(10):\n",
" response=get_completion(\"一千字评价一下下三国演义中的曹操\")\n",
" total_words += len(response.choices[0].message[\"content\"])\n",
" print(f\"第{i}次返回的字数:\", len(response.choices[0].message[\"content\"]))\n",
"print(f\"平均每次返回的字数: {total_words / 10}\")\n",
"print(f\"平均每秒输出的字数：{total_words // (time.time() - start_time)}\")"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# openai的文档（https://platform.openai.com/docs/api-reference/completions/object）中提到，\n",
"# openai.ChatCompletion.create返回对象中包含usage字段，该字段统计了input promot和output以及两者之和的token数量。但是实际打印中该字段已经被移除。\n",
"\n",
"# print(response)\n",
"\n",
"{\n",
" \"model\": \"chatglm2-6b-model\",\n",
" \"object\": \"chat.completion\",\n",
" \"choices\": [\n",
" {\n",
" \"index\": 0,\n",
" \"message\": {\n",
" \"role\": \"assistant\",\n",
" \"content\": \"\\u4e0b\\u4e09\\u56fd\\u6f14\\u4e49\\u4e2d\\u7684\\u66f9\\u64cd\\uff0c\\u4f5c\\u4e3a\\u4e1c\\u6c49\\u672b\\u5e74\\u7684\\u653f\\u6cbb\\u5bb6\\u548c\\u519b\\u4e8b\\u5bb6\\uff0c\\u662f\\u4e00\\u4f4d\\u6781\\u5177\\u4e89\\u8bae\\u7684\\u5386\\u53f2\\u4eba\\u7269\\u3002\\u4e00\\u65b9\\u9762\\uff0c\\u66f9\\u64cd\\u8eab\\u7ecf\\u767e\\u6218\\uff0c\\u7edf\\u4e00\\u5317\\u65b9\\uff0c\\u4e3a\\u56fd\\u5bb6\\u7acb\\u4e0b\\u4e86\\u8d6b\\u8d6b\\u6218\\u529f\\uff1b\\u53e6\\u4e00\\u65b9\\u9762\\uff0c\\u66f9\\u64cd\\u53c8\\u662f\\u4e00\\u4f4d\\u6781\\u5177\\u8c0b\\u7565\\u548c\\u51b7\\u9177\\u7684\\u7edf\\u6cbb\\u8005\\uff0c\\u7ecf\\u5e38\\u65bd\\u884c\\u6b8b\\u9177\\u7684\\u5211\\u7f5a\\uff0c\\u5bfc\\u81f4\\u6c11\\u4f17\\u5bf9\\u5176\\u6028\\u58f0\\u8f7d\\u9053\\u3002\\n\\n\\u66f9\\u64cd\\u5728\\u4e09\\u56fd\\u65f6\\u671f\\u7684\\u6218\\u4e89\\u7ecf\\u5386\\u4e2d\\uff0c\\u5c55\\u73b0\\u51fa\\u4e86\\u51fa\\u8272\\u7684\\u519b\\u4e8b\\u624d\\u80fd\\u548c\\u6307\\u6325\\u80fd\\u529b\\u3002\\u4ed6\\u7387\\u9886\\u7684\\u90e8\\u961f\\u5728\\u5b98\\u6e21\\u3001\\u8d64\\u58c1\\u7b49\\u6218\\u5f79\\u4e2d\\u53d6\\u5f97\\u91cd\\u5927\\u80dc\\u5229\\uff0c\\u5bf9\\u5f53\\u65f6\\u7684\\u653f\\u6cbb\\u5c40\\u52bf\\u4ea7\\u751f\\u4e86\\u91cd\\u5927\\u5f71\\u54cd\\u3002\\u540c\\u65f6\\uff0c\\u66f9\\u64cd\\u8fd8\\u662f\\u4e00\\u4f4d\\u51fa\\u8272\\u7684\\u653f\\u6cbb\\u5bb6\\uff0c\\u5728\\u6cbb\\u7406\\u56fd\\u5bb6\\u65b9\\u9762\\u6709\\u7740\\u72ec\\u7279\\u7684\\u89c1\\u89e3\\uff0c\\u4e3a\\u56fd\\u5bb6\\u7684\\u53d1\\u5c55\\u505a\\u51fa\\u4e86\\u91cd\\u8981\\u8d21\\u732e\\u3002\\n\\n\\u7136\\u800c\\uff0c\\u66f9\\u64cd\\u7684\\u6b8b\\u9177\\u7edf\\u6cbb\\u548c\\u8840\\u8165\\u624b\\u6bb5\\u4e5f\\u96be\\u4ee5\\u9003\\u8131\\u5386\\u53f2\\u60a0\\u60a0\\u4f17\\u53e3\\u7684\\u5ba1\\u5224\\u3002\\u5728\\u4e09\\u56fd\\u65f6\\u671f\\uff0c\\u66f9\\u64cd\\u7ecf\\u5e38\\u65bd\\u884c\\u6781\\u5176\\u6b8b\\u9177\\u7684\\u5211\\u7f5a\\uff0c\\u5982\\u65a9\\u9996\\u3001\\u7ede\\u5211\\u7b49\\uff0c\\u6765\\u5de9\\u56fa\\u81ea\\u5df1\\u7684\\u7edf\\u6cbb\\u5730\\u4f4d\\u3002\\u8fd9\\u4e9b\\u5211\\u7f5a\\u4e0d\\u4ec5\\u5bf9\\u7f6a\\u72af\\u672c\\u4eba\\u9020\\u6210\\u4e86\\u6781\\u5927\\u7684\\u75db\\u82e6\\uff0c\\u4e5f\\u5bf9\\u666e\\u901a\\u767e\\u59d3\\u5e26\\u6765\\u4e86\\u4e0d\\u53ef\\u627f\\u53d7\\u7684\\u60ca\\u6050\\u548c\\u60b2\\u4f24\\u3002\\n\\n\\u603b\\u4e4b\\uff0c\\u4e0b\\u4e09\\u56fd\\u6f14\\u4e49\\u4e2d\\u7684\\u66f9\\u64cd\\u662f\\u4e00\\u4f4d\\u6781\\u5177\\u4e89\\u8bae\\u7684\\u5386\\u53f2\\u4eba\\u7269\\u3002\\u4ed6\\u65e2\\u6709\\u51fa\\u8272\\u7684\\u519b\\u4e8b\\u624d\\u80fd\\u548c\\u653f\\u6cbb\\u667a\\u6167\\uff0c\\u53c8\\u6709\\u6b8b\\u9177\\u7edf\\u6cbb\\u548c\\u8840\\u8165\\u624b\\u6bb5\\u3002\\u5728\\u8bc4\\u4ef7\\u66f9\\u64cd\\u65f6\\uff0c\\u6211\\u4eec\\u9700\\u8981\\u5168\\u9762\\u5ba2\\u89c2\\u5730\\u770b\\u5f85\\u4ed6\\u7684\\u5386\\u53f2\\u4f5c\\u7528\\u548c\\u8d1f\\u9762\\u5f71\\u54cd\\u3002\"\n",
" },\n",
" \"finish_reason\": \"stop\"\n",
" }\n",
" ],\n",
" \"created\": 1693113050\n",
"}"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# 可参考 https://github.com/openai/openai-cookbook/blob/main/examples/How_to_count_tokens_with_tiktoken.ipynb 来计算Token数量\n",
"# 扩展阅读：https://zhuanlan.zhihu.com/p/626593576 \n",
"\n",
"# To-Do:\n",
"# 可使用tiktoken库的来计算Token数量\n",
"\n",
"import tiktoken\n",
"\n",
"def num_tokens_from_string(string: str, encoding_name: str) -> int:\n",
" \"\"\"Returns the number of tokens in a text string.\"\"\"\n",
" encoding = tiktoken.get_encoding(encoding_name)\n",
" num_tokens = len(encoding.encode(string))\n",
" return num_tokens\n",
"\n",
"num_tokens_from_string(\"tiktoken is great!\", \"cl100k_base\")\n",
"\n",
"# p50k_base和cl100k_base两种不同的encoding方式，cl100k_base的Token数量比p50k_base多\n",
"num_tokens_from_string(response.choices[0].message[\"content\"], \"p50k_base\")\n",
"num_tokens_from_string(response.choices[0].message[\"content\"], \"cl100k_base\")"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "chatGLM",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.9.16"
},
"orig_nbformat": 4
},
"nbformat": 4,
"nbformat_minor": 2
}
Write Preview
Loading…
Cancel
Save