From cccd216977321c447553eab80ff05e8829c76417 Mon Sep 17 00:00:00 2001 From: x54-729 Date: Thu, 2 Nov 2023 23:17:30 +0800 Subject: [PATCH] fix import of tools/tokenizer.py --- tools/tokenizer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/tokenizer.py b/tools/tokenizer.py index fc3800e..8e0a0d0 100644 --- a/tools/tokenizer.py +++ b/tools/tokenizer.py @@ -8,7 +8,7 @@ import numpy as np current_dir = os.path.dirname(os.path.abspath(__file__)) model_path = os.path.join(current_dir, "V7_sft.model") sys.path.append(os.path.join(current_dir, "transformers")) -from tokenization_internlm import InternLMTokenizer +from internlm_model import InternLMTokenizer tokenizer = InternLMTokenizer(vocab_file=model_path)