mirror of https://github.com/InternLM/InternLM
test(workflow): add basic model test (#650)
Co-authored-by: kkscilife <wangmengke@pjlab.org.cn>pull/663/head
kkscilife
GitHub
parent
f395c4b5d1
commit
c1ecc0d3d5
|
|
@ -0,0 +1,57 @@
|
|||
name: basic-model-tests-daily
|
||||
on:
|
||||
workflow_dispatch:
|
||||
schedule:
|
||||
- cron: '48 19 * * *'
|
||||
env:
|
||||
WORKSPACE_PREFIX: $(echo $GITHUB_WORKSPACE |cut -d '/' -f 1-4)
|
||||
SLURM_PARTITION: llm_s
|
||||
CONDA_BASE_ENV: internlm-model-base
|
||||
|
||||
jobs:
|
||||
HF_model:
|
||||
runs-on: [t_cluster]
|
||||
steps:
|
||||
- name: mask env
|
||||
run: |
|
||||
echo "::add-mask::${{env.WORKSPACE_PREFIX}}"
|
||||
echo "::add-mask::$path_prefix"
|
||||
- uses: actions/checkout@v3
|
||||
|
||||
- name: load_hf_model
|
||||
run: |
|
||||
conda create -n internlm-model-latest --clone ${CONDA_BASE_ENV}
|
||||
source activate internlm-model-latest
|
||||
# TODO:test other version of transformers
|
||||
pip install transformers
|
||||
pip install sentencepiece
|
||||
srun -p ${SLURM_PARTITION} --kill-on-bad-exit=1 --job-name=${GITHUB_RUN_ID}-${GITHUB_JOB} --gpus-per-task=2 pytest -s -v --color=yes ./tests/test_hf_model.py
|
||||
conda deactivate
|
||||
|
||||
clear_env:
|
||||
if: ${{ !cancelled() }}
|
||||
needs: [HF_model]
|
||||
runs-on: [t_cluster]
|
||||
timeout-minutes: 10
|
||||
steps:
|
||||
- name: mask env
|
||||
run: |
|
||||
echo "::add-mask::${{env.WORKSPACE_PREFIX}}"
|
||||
echo "::add-mask::$path_prefix"
|
||||
|
||||
- name: remove_env
|
||||
run: |
|
||||
conda env remove --name internlm-model-latest
|
||||
|
||||
notify_to_feishu:
|
||||
if: ${{ always() && !cancelled() && contains(needs.*.result, 'failure') && (github.ref_name == 'develop' || github.ref_name == 'main') }}
|
||||
needs: [HF_model,clear_env]
|
||||
runs-on: [t_cluster]
|
||||
steps:
|
||||
- name: mask env
|
||||
run: |
|
||||
echo "::add-mask::${{env.WORKSPACE_PREFIX}}"
|
||||
echo "::add-mask::$path_prefix"
|
||||
- name: notify
|
||||
run: |
|
||||
curl -X POST -H "Content-Type: application/json" -d '{"msg_type":"post","content":{"post":{"zh_cn":{"title":"Internlm GitHubAction Failed","content":[[{"tag":"text","text":""},{"tag":"a","text":"Please click here for details ","href":"https://github.com/'${{ github.repository }}'/actions/runs/'${GITHUB_RUN_ID}'"},{"tag":"at","user_id":"'${{ secrets.USER_ID }}'"}]]}}}}' ${{ secrets.WEBHOOK_URL }}
|
||||
|
|
@ -0,0 +1,79 @@
|
|||
import pytest
|
||||
import torch
|
||||
from transformers import AutoModelForCausalLM, AutoTokenizer
|
||||
|
||||
prompts = ["你好", "what's your name"]
|
||||
|
||||
|
||||
def assert_model(response):
|
||||
assert len(response) != 0
|
||||
assert "UNUSED_TOKEN" not in response
|
||||
|
||||
|
||||
class TestChat:
|
||||
"""
|
||||
Test cases for chat model.
|
||||
"""
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model_name",
|
||||
[
|
||||
"internlm/internlm2-chat-7b",
|
||||
"internlm/internlm2-chat-7b-sft",
|
||||
],
|
||||
)
|
||||
def test_demo_default(self, model_name):
|
||||
tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
|
||||
# Set `torch_dtype=torch.float16` to load model in float16, otherwise
|
||||
# it will be loaded as float32 and might cause OOM Error.
|
||||
model = AutoModelForCausalLM.from_pretrained(
|
||||
model_name, torch_dtype=torch.float16, trust_remote_code=True
|
||||
).cuda()
|
||||
model = model.eval()
|
||||
for prompt in prompts:
|
||||
response, history = model.chat(tokenizer, prompt, history=[])
|
||||
print(response)
|
||||
assert_model(response)
|
||||
|
||||
for prompt in prompts:
|
||||
length = 0
|
||||
for response, history in model.stream_chat(tokenizer, prompt, history=[]):
|
||||
print(response[length:], flush=True, end="")
|
||||
length = len(response)
|
||||
assert_model(response)
|
||||
|
||||
|
||||
class TestBase:
|
||||
"""
|
||||
Test cases for base model.
|
||||
"""
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model_name",
|
||||
[
|
||||
"internlm/internlm2-7b",
|
||||
"internlm/internlm2-base-7b",
|
||||
],
|
||||
)
|
||||
def test_demo_default(self, model_name):
|
||||
tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
|
||||
# Set `torch_dtype=torch.float16` to load model in float16, otherwise
|
||||
# it will be loaded as float32 and might cause OOM Error.
|
||||
model = AutoModelForCausalLM.from_pretrained(
|
||||
model_name, torch_dtype=torch.float16, trust_remote_code=True
|
||||
).cuda()
|
||||
for prompt in prompts:
|
||||
inputs = tokenizer(prompt, return_tensors="pt")
|
||||
for k, v in inputs.items():
|
||||
inputs[k] = v.cuda()
|
||||
gen_kwargs = {
|
||||
"max_length": 128,
|
||||
"top_p": 10,
|
||||
"temperature": 1.0,
|
||||
"do_sample": True,
|
||||
"repetition_penalty": 1.0,
|
||||
}
|
||||
output = model.generate(**inputs, **gen_kwargs)
|
||||
output = tokenizer.decode(output[0].tolist(), skip_special_tokens=True)
|
||||
print(output)
|
||||
assert_model(output)
|
||||
Loading…
Reference in New Issue