github
/
ColossalAI
mirror of https://github.com/hpcaitech/ColossalAI
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

[coati] add repetition_penalty for inference (#3294)

pull/3295/head
ver217 2023-03-29 01:18:45 +08:00 committed by GitHub
parent a88ed0f83a
commit ce2cafae76
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 2 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
applications/Chat/inference/server.py
View File

@ -27,6 +27,7 @@ class GenerationTaskReq(BaseModel):
top_k: Optional[int] = Field(default=None, gt=0, example=50) top_k: Optional[int] = Field(default=None, gt=0, example=50)
top_p: Optional[float] = Field(default=None, gt=0.0, lt=1.0, example=0.5) top_p: Optional[float] = Field(default=None, gt=0.0, lt=1.0, example=0.5)
temperature: Optional[float] = Field(default=None, gt=0.0, lt=1.0, example=0.7) temperature: Optional[float] = Field(default=None, gt=0.0, lt=1.0, example=0.7)
repetition_penalty: Optional[float] = Field(default=None, gt=1.0, example=1.2)
limiter = Limiter(key_func=get_remote_address) limiter = Limiter(key_func=get_remote_address)
@ -55,6 +56,7 @@ app.add_middleware(
def generate_streamingly(prompt, max_new_tokens, top_k, top_p, temperature): def generate_streamingly(prompt, max_new_tokens, top_k, top_p, temperature):
inputs = {k: v.cuda() for k, v in tokenizer(prompt, return_tensors="pt").items()} inputs = {k: v.cuda() for k, v in tokenizer(prompt, return_tensors="pt").items()}
#TODO(ver217): streaming generation does not support repetition_penalty now
model_kwargs = { model_kwargs = {
'max_generate_tokens': max_new_tokens, 'max_generate_tokens': max_new_tokens,
'early_stopping': True, 'early_stopping': True,