InternLM/tools/alpaca_tokenizer.py

import argparse
import json
import os.path as osp
from pathlib import Path

import numpy as np
import sentencepiece as spm
from tqdm import tqdm


def process(dataset_path, sp_model):
    """Process data sample from input dataset

    Args:
        dataset_path (str): Path of dataset json file.
        sp_model (str): Path of tokenizer.

    Yields:
        tuple: dumped processed data sample and length of tokens.
    """

    dataset = json.load(open(dataset_path))

    for data in dataset:
        yield tokenize(get_chat_format_data(data), sp_model)


def get_chat_format_data(ori_data):
    """Format original data

    Args:
        ori_data (dict): input data sample.

    Returns:
        dict: data sample with chat format.
    """
    input_str = ori_data["input"]
    instruction_str = ori_data["instruction"]
    output_str = ori_data["output"]
    data = dict()
    if input_str != "":
        data["user"] = f"<|User|>:{instruction_str}\n{input_str}"
    else:
        data["user"] = f"<|User|>:{instruction_str}"
    data["bot"] = f"<|Bot|>:{output_str}"
    return data


def tokenize(sample, sp_model):
    """Tokenize input dataset

    Args:
        sample (dict): Input data sample.
        sp_model (str): Path of tokenizer.

    Returns:
        tuple: dumped processed data sample and length of tokens.
    """
    special_tokens_map = {"<eoh>": 103167, "<eoa>": 103166, "nl_id": 13}
    token_ids = [sp_model.bos_id()]
    human_s = sample["user"]
    ass_s = sample["bot"]

    human_ids = sp_model.encode(human_s) + [special_tokens_map["<eoh>"], special_tokens_map["nl_id"]]
    human_ids_ignore = [-token_id for token_id in human_ids]

    ass_template_ids = sp_model.encode("<|Bot|>:")
    ass_template_ids_ignore = [-token_ids for token_ids in ass_template_ids]
    ass_ids = (
        ass_template_ids_ignore
        + sp_model.encode(ass_s[8:])
        + [special_tokens_map["<eoa>"], special_tokens_map["nl_id"]]
    )

    token_ids += human_ids_ignore + ass_ids
    if len(token_ids) > 2047:
        token_ids = token_ids[:2047]
    token_ids += [sp_model.eos_id()]
    line = str.encode(json.dumps({"tokens": token_ids}) + "\n")
    return line, len(token_ids)


def dump_bin_meta_bin(samples, path, split_ratio=0.1):
    """Dump processed dataset

    Args:
        samples (dict): Input data sample.
        path (str): Path for output dataset.
        split_ratio (float): Ratio for validation dataset splitting.
            Default to: 0.1.

    Returns:
        tuple: number of train/valid tokens of processed dataset,
            number of train/valid samples of processed dataset.
    """

    train_path = osp.join(path, "train/en/")
    valid_path = osp.join(path, "valid/en/")
    train_dir = Path(train_path)
    valid_dir = Path(valid_path)
    train_dir.mkdir(exist_ok=True, parents=True)
    valid_dir.mkdir(exist_ok=True, parents=True)
    train_f = open(train_dir.joinpath("dataset.bin"), "wb")
    valid_f = open(valid_dir.joinpath("dataset.bin"), "wb")

    train_tokens = 0
    valid_tokens = 0
    last_train_position = 0
    last_valid_position = 0
    train_samples = 0
    valid_samples = 0
    train_meta = []
    valid_meta = []

    sample_length = len(samples)
    np.random.seed(0)
    valid_indices = np.random.choice(range(sample_length), int(sample_length * split_ratio)).tolist()

    count = -1
    for line, token_num in samples:
        count += 1
        if count in valid_indices:
            valid_tokens += token_num
            valid_f.write(line)
            valid_meta.append((last_valid_position, token_num))
            last_valid_position += len(line)
            valid_samples += 1
        else:
            train_tokens += token_num
            train_f.write(line)
            train_meta.append((last_train_position, token_num))
            last_train_position += len(line)
            train_samples += 1

    train_f.close()
    valid_f.close()
    np.save(open(train_dir.joinpath("dataset.bin.meta"), "wb"), train_meta)
    np.save(open(valid_dir.joinpath("dataset.bin.meta"), "wb"), valid_meta)

    return train_tokens, valid_tokens, train_samples, valid_samples


if __name__ == "__main__":

    parser = argparse.ArgumentParser()
    parser.add_argument("dataset_path", type=str, help="path of dataset json file")
    parser.add_argument("output_path", type=str, help="path of processed dataset")
    parser.add_argument("tokenizer_path", type=str, help="path of tokenizer")
    parser.add_argument("--split_ratio", type=float, default=0.1, help="ratio for validation dataset splitting")

    args = parser.parse_args()
    sp_model = spm.SentencePieceProcessor(model_file=args.tokenizer_path)
    split_ratio = args.split_ratio
    samples = []

    dataset = process(args.dataset_path, sp_model)
    for sample in tqdm(dataset):
        samples.append(sample)

    train_tokens, valid_tokens, train_samples, valid_samples = dump_bin_meta_bin(
        samples, args.output_path, args.split_ratio
    )
    print(f"number of train dataset: {train_samples}, number of train dataset token: {train_tokens}")
    print(f"number of validation dataset: {valid_samples}, number of validation dataset token: {valid_tokens}")