From a9fc0184446fba7f4f27addf519fea0b371df83a Mon Sep 17 00:00:00 2001 From: rainatam Date: Thu, 6 Apr 2023 15:16:36 +0800 Subject: [PATCH] Update evaluation results and bleu score function --- ptuning/README.md | 39 ++++++++++++++++++++++++++++++++++++++- ptuning/main.py | 4 ++-- 2 files changed, 40 insertions(+), 3 deletions(-) diff --git a/ptuning/README.md b/ptuning/README.md index 4978dc4..ca1fc73 100644 --- a/ptuning/README.md +++ b/ptuning/README.md @@ -52,10 +52,46 @@ bash evaluate.sh * Input: 类型#裙\*版型#显瘦\*风格#文艺\*风格#简约\*图案#印花\*图案#撞色\*裙下摆#压褶\*裙长#连衣裙\*裙领型#圆领 * Label: 文艺个性的印花连衣裙,藏青色底蕴,低调又大气,撞色太阳花分布整个裙身,绚丽而美好,带来时尚减龄的气质。基础款的舒适圆领,简约不失大方,勾勒精致脸庞。领后是一粒包布扣固定,穿脱十分方便。前片立体的打褶设计,搭配后片压褶的做工,增添层次和空间感,显瘦又有型。 - * Output[微调前]: 类型#裙*版型#显瘦*风格#文艺*风格#简约*图案#印花*图案#撞色*裙下摆#压褶*裙长#连衣裙*裙领型#圆领 1\. 连衣裙:简约风格,裙长为膝盖以上,裙领型为圆领。2\. 裙下摆:压褶设计,使裙摆呈现出流畅的褶皱效果。3\. 裙领型:裙领型为圆领,使穿上连衣裙后更加有型。4\. 版型:采用显瘦设计,让连衣裙看起来更加苗条。5\. 风格:文艺风格,让连衣裙更加有内涵和品味。6\. 图案:印花设计,在连衣裙上印有独特的图案。7\. 撞色:采用撞色设计,让连衣裙在色彩上更加鲜明、富有层次感。 * Output[微调后]: 这是一款文艺范的连衣裙,以印花为元素,采用简约的印花,既能够突出文艺气质,又能够展现简约风。在印花的同时又有领子和裙摆的压褶设计,更加凸显文艺气质。简约而不会过于单调,搭配出街,穿着十分舒适。 +### 评估结果 + +| | P-tuning v2 | LoRA | +| ------- | ----------- | ----- | +| BLEU-4 | 7.71 | 6.13 | +| Rouge-1 | 31.35 | 28.36 | +| Rouge-2 | 7.19 | 4.38 | +| Rouge-l | 25.17 | 17.54 | + +#### 实验设置 + + ``` +max_source_length=64 +max_target_length=64 +per_device_train_batch_size=1 +gradient_accumulation_steps=16 +max_steps=3000 + ``` + +##### P-tuning v2 + +``` +pre_seq_len=128 +learning_rate=2e-2 +quantization_bit=4 +``` + +##### LoRA + +``` +learning_rate=5e-4 +``` + +实现采用的是 [simple_thu_chatglm6b](https://github.com/yuanzhoulvpi2017/zero_nlp/tree/main/simple_thu_chatglm6b) + + + ## 模型部署 将对应的demo或代码中的`THUDM/chatglm-6b`换成经过 P-Tuning 微调之后 checkpoint 的地址（在示例中为 `./output/adgen-chatglm-6b-pt-8-1e-2/checkpoint-3000`）。注意，目前的微调还不支持多轮数据，所以只有对话第一轮的回复是经过微调的。 @@ -77,3 +113,4 @@ bash evaluate.sh year={2022} } ``` + diff --git a/ptuning/main.py b/ptuning/main.py index 112c9ca..fbf3924 100644 --- a/ptuning/main.py +++ b/ptuning/main.py @@ -27,7 +27,7 @@ import numpy as np from datasets import load_dataset import jieba from rouge_chinese import Rouge -from nltk.translate.bleu_score import sentence_bleu +from nltk.translate.bleu_score import sentence_bleu, SmoothingFunction import transformers from transformers import ( @@ -293,7 +293,7 @@ def main(): for k, v in result.items(): score_dict[k].append(round(v["f"] * 100, 4)) - bleu_score = sentence_bleu([list(label)], list(pred)) + bleu_score = sentence_bleu([list(label)], list(pred), smoothing_function=SmoothingFunction().method3) score_dict["bleu-4"].append(round(bleu_score * 100, 4)) for k, v in score_dict.items():