From b6ec337f3dbede0f90a27cf354d618bf9ae23e64 Mon Sep 17 00:00:00 2001 From: Tong Li Date: Thu, 14 Nov 2024 08:28:02 +0000 Subject: [PATCH] update tokenize function --- applications/ColossalChat/coati/dataset/tokenization_utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/applications/ColossalChat/coati/dataset/tokenization_utils.py b/applications/ColossalChat/coati/dataset/tokenization_utils.py index 533d0acad..0548a1454 100755 --- a/applications/ColossalChat/coati/dataset/tokenization_utils.py +++ b/applications/ColossalChat/coati/dataset/tokenization_utils.py @@ -431,8 +431,8 @@ def tokenize_process_reward( decoded_label = tokenizer.decode(label, skip_special_tokens=False) return { - "input": tokenized, - "label": label, + "input_ids": tokenized, + "labels": label, "loss_mask": loss_mask, "decoded_input": decoded_input, "decoded_label": decoded_label,