From 508711cc97278f6c51f2684fac446889c2cd4d44 Mon Sep 17 00:00:00 2001 From: djsaber <60215276+djsaber@users.noreply.github.com> Date: Tue, 26 Dec 2023 16:52:06 +0800 Subject: [PATCH] Update modeling_internlm.py fixed the issue that the HF model spontaneously conducted multiple rounds of Q&A and stream_chat method generates garbled characters --- tools/transformers/modeling_internlm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/transformers/modeling_internlm.py b/tools/transformers/modeling_internlm.py index b817f69..fed7013 100644 --- a/tools/transformers/modeling_internlm.py +++ b/tools/transformers/modeling_internlm.py @@ -861,7 +861,7 @@ class InternLMForCausalLM(InternLMPreTrainedModel): self.chche.extend(value.tolist()) token = self.tokenizer.decode(self.chche, skip_special_tokens=True) - if " " in token and len(token) <= 5: + if "�" in token and len(token) <= 5: return if token.strip() != "":