From 7cfea534e7c7191d80a3b6db61fa335bc229ff7e Mon Sep 17 00:00:00 2001
From: Shaoyuan Xie <66255889+Daniel-xsy@users.noreply.github.com>
Date: Thu, 10 Aug 2023 17:53:46 +0800
Subject: [PATCH] [feat]: add pal reasoning script (#163)

* [Feat] Add PAL inference script

* Update README.md

* Update tools/README.md

Co-authored-by: BigDong <yudongwang1226@gmail.com>

* Update tools/pal_inference.py

Co-authored-by: BigDong <yudongwang1226@gmail.com>

* Update pal script

* Update README.md

* restore .ore-commit-config.yaml

* Update tools/README.md

Co-authored-by: BigDong <yudongwang1226@gmail.com>

* Update tools/README.md

Co-authored-by: BigDong <yudongwang1226@gmail.com>

* Update pal inference script

* Update READMD.md

* Update internlm/utils/interface.py

Co-authored-by: Wenwei Zhang <40779233+ZwwWayne@users.noreply.github.com>

* Update pal script

* Update pal script

* Update script

* Add docstring

* Update format

* Update script

* Update script

* Update script

---------

Co-authored-by: BigDong <yudongwang1226@gmail.com>
Co-authored-by: Wenwei Zhang <40779233+ZwwWayne@users.noreply.github.com>
---
 internlm/utils/interface.py              | 138 ++++++++++
 internlm/utils/simple_memory_profiler.py |   8 +-
 internlm/utils/timeout.py                |  26 ++
 tools/README.md                          |  61 ++++-
 tools/README_EN.md                       |  62 ++++-
 tools/pal_inference.py                   | 320 +++++++++++++++++++++++
 6 files changed, 607 insertions(+), 8 deletions(-)
 create mode 100644 internlm/utils/interface.py
 create mode 100644 internlm/utils/timeout.py
 create mode 100644 tools/pal_inference.py

diff --git a/internlm/utils/interface.py b/internlm/utils/interface.py
new file mode 100644
index 0000000..22b3743
--- /dev/null
+++ b/internlm/utils/interface.py
@@ -0,0 +1,138 @@
+import copy
+import warnings
+from dataclasses import dataclass
+from typing import Callable, List, Optional
+
+import torch
+from transformers import AutoModel, AutoTokenizer
+from transformers.generation.utils import LogitsProcessorList, StoppingCriteriaList
+from transformers.utils import logging
+
+logger = logging.get_logger(__name__)
+
+
+@dataclass
+class GenerationConfig:
+    max_length: Optional[int] = None
+    top_p: Optional[float] = None
+    temperature: Optional[float] = None
+    do_sample: Optional[bool] = True
+    repetition_penalty: Optional[float] = 1.0
+
+
+@torch.inference_mode()
+def generation_iterator(
+    model: AutoModel,
+    tokenizer: AutoTokenizer,
+    prompt: str,
+    generation_config: Optional[GenerationConfig] = None,
+    logits_processor: Optional[LogitsProcessorList] = None,
+    stopping_criteria: Optional[StoppingCriteriaList] = None,
+    prefix_allowed_tokens_fn: Optional[Callable[[int, torch.Tensor], List[int]]] = None,
+    additional_eos_token_id: Optional[int] = None,
+    **kwargs,
+):
+    inputs = tokenizer([prompt], padding=True, return_tensors="pt")
+    input_length = len(inputs["input_ids"][0])
+    for k, v in inputs.items():
+        inputs[k] = v.cuda()
+    input_ids = inputs["input_ids"]
+    input_ids_seq_length = input_ids.shape[-1]
+    if generation_config is None:
+        generation_config = model.generation_config
+    generation_config = copy.deepcopy(generation_config)
+    model_kwargs = generation_config.update(**kwargs)
+    eos_token_id = generation_config.eos_token_id
+    if isinstance(eos_token_id, int):
+        eos_token_id = [eos_token_id]
+    if additional_eos_token_id is not None:
+        eos_token_id.append(additional_eos_token_id)
+    has_default_max_length = kwargs.get("max_length") is None and generation_config.max_length is not None
+    if has_default_max_length and generation_config.max_new_tokens is None:
+        warnings.warn(
+            f"Using `max_length`'s default ({generation_config.max_length}) to control the generation length. "
+            "This behaviour is deprecated and will be removed from the config in v5 of Transformers -- we"
+            " recommend using `max_new_tokens` to control the maximum length of the generation.",
+            UserWarning,
+        )
+    elif generation_config.max_new_tokens is not None:
+        generation_config.max_length = generation_config.max_new_tokens + input_ids_seq_length
+        if not has_default_max_length:
+            logger.warning(
+                "Both `max_new_tokens` (={%s}) and `max_length`(="
+                "{%s}) seem to have been set. `max_new_tokens` will take precedence. "
+                "Please refer to the documentation for more information. "
+                "(https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)",
+                generation_config.max_new_tokens,
+                generation_config.max_length,
+            )
+
+    if input_ids_seq_length >= generation_config.max_length:
+        input_ids_string = "input_ids"
+        logger.warning(
+            "Input length of {%s} is {%s}, but `max_length` is set to"
+            " {%s}. This can lead to unexpected behavior. You should consider"
+            " increasing `max_new_tokens`.",
+            input_ids_string,
+            input_ids_seq_length,
+            generation_config.max_length,
+        )
+
+    # 2. Set generation parameters if not already defined
+    logits_processor = logits_processor if logits_processor is not None else LogitsProcessorList()
+    stopping_criteria = stopping_criteria if stopping_criteria is not None else StoppingCriteriaList()
+
+    logits_processor = model._get_logits_processor(
+        generation_config=generation_config,
+        input_ids_seq_length=input_ids_seq_length,
+        encoder_input_ids=input_ids,
+        prefix_allowed_tokens_fn=prefix_allowed_tokens_fn,
+        logits_processor=logits_processor,
+    )
+
+    stopping_criteria = model._get_stopping_criteria(
+        generation_config=generation_config, stopping_criteria=stopping_criteria
+    )
+    logits_warper = model._get_logits_warper(generation_config)
+
+    unfinished_sequences = input_ids.new(input_ids.shape[0]).fill_(1)
+    scores = None
+    while True:
+        model_inputs = model.prepare_inputs_for_generation(input_ids, **model_kwargs)
+        # forward pass to get next token
+        outputs = model(
+            **model_inputs,
+            return_dict=True,
+            output_attentions=False,
+            output_hidden_states=False,
+        )
+
+        next_token_logits = outputs.logits[:, -1, :]
+
+        # pre-process distribution
+        next_token_scores = logits_processor(input_ids, next_token_logits)
+        next_token_scores = logits_warper(input_ids, next_token_scores)
+
+        # sample
+        probs = next_token_scores.softmax(dim=-1)
+        if generation_config.do_sample:
+            next_tokens = torch.multinomial(probs, num_samples=1).squeeze(1)
+        else:
+            next_tokens = torch.argmax(probs, dim=-1)
+
+        # update generated ids, model inputs, and length for next step
+        input_ids = torch.cat([input_ids, next_tokens[:, None]], dim=-1)
+        model_kwargs = model._update_model_kwargs_for_generation(outputs, model_kwargs, is_encoder_decoder=False)
+        unfinished_sequences = unfinished_sequences.mul((min(next_tokens != i for i in eos_token_id)).long())
+
+        output_token_ids = input_ids[0].cpu().tolist()
+        output_token_ids = output_token_ids[input_length:]
+        for each_eos_token_id in eos_token_id:
+            if output_token_ids[-1] == each_eos_token_id:
+                output_token_ids = output_token_ids[:-1]
+        response = tokenizer.decode(output_token_ids)
+
+        yield response
+        # stop when each sentence is finished, or if we exceed the maximum length
+        if unfinished_sequences.max() == 0 or stopping_criteria(input_ids, scores):
+            break
diff --git a/internlm/utils/simple_memory_profiler.py b/internlm/utils/simple_memory_profiler.py
index 7fd8b28..4ca6679 100644
--- a/internlm/utils/simple_memory_profiler.py
+++ b/internlm/utils/simple_memory_profiler.py
@@ -218,9 +218,7 @@ class SimpleMemoryProfiler:
         # Calculate static optimizer state cuda memory
         self._os_params_mem_state = SimpleMemState("os_params_mem")
         self._os_state_mem_state = SimpleMemState("os_state_mem")
-        self._calc_tensor_group_memory(
-            self._os_params_mem_state, [(k, v) for k, v in enumerate(self._optimizer.param_groups)]
-        )
+        self._calc_tensor_group_memory(self._os_params_mem_state, list(enumerate(self._optimizer.param_groups)))
 
         # Generate the first memory record
         self.point(create=True)
@@ -302,9 +300,7 @@ class SimpleMemoryProfiler:
 
         # Update os state memory usage
         self._os_state_mem_state = SimpleMemState("os_state_mem")
-        self._calc_tensor_group_memory(
-            self._os_state_mem_state, [(k, v) for k, v in self._optimizer.state_dict()["state"].items()]
-        )
+        self._calc_tensor_group_memory(self._os_state_mem_state, list(self._optimizer.state_dict()["state"].items()))
 
         if not self._stoped:
             # Do we need to print os_state_layout every time? Is it always constant?
diff --git a/internlm/utils/timeout.py b/internlm/utils/timeout.py
new file mode 100644
index 0000000..07a0911
--- /dev/null
+++ b/internlm/utils/timeout.py
@@ -0,0 +1,26 @@
+import signal
+
+
+class Timeout:
+    """Timer to execute code
+
+    Adapted from https://github.com/reasoning-machines/pal
+
+    Args:
+        seconds (float): The maximum seconds to execute code
+        error_message (str)
+    """
+
+    def __init__(self, seconds=1, error_message="Timeout"):
+        self.seconds = seconds
+        self.error_message = error_message
+
+    def timeout_handler(self, signum, frame):
+        raise TimeoutError(self.error_message)
+
+    def __enter__(self):
+        signal.signal(signal.SIGALRM, self.timeout_handler)
+        signal.alarm(self.seconds)
+
+    def __exit__(self, error_type, value, traceback):
+        signal.alarm(0)
diff --git a/tools/README.md b/tools/README.md
index f3ba385..ae7d196 100644
--- a/tools/README.md
+++ b/tools/README.md
@@ -1,4 +1,5 @@
 本目录提供辅助模型训练的一些工具，文件结构如下所示：
+
 ```bash
 ├── transformers  # 适配hugging face的transformers的一些工具
 │   ├── configuration_internlm.py  # config适配工具
@@ -9,9 +10,11 @@
 ```
 
 # tokenizer.py
+
 生成原始数据的`bin`和`meta`文件需要使用`tokenizer`，我们通过在`tools/tokenizer.py`中指定模型参数路径的方式来导入tokenizer模型。目前我们提供了`V7_sft.model`来生成tokens。若想使用不同的模型，可直接修改`tokernizer.py`中的模型参数路径。
 
 可以运行以下命令生成原始数据对应的`bin`和`meta`文件，其中参数`text_input_path`表示原始文本数据路径，目前支持`txt`、`json`和`jsonl`三种输入格式，`bin_output_path`表示生成的`bin`文件的保存路径。
+
 ```bash
 $ python tools/tokenizer.py --text_input_path your_input_text_path --bin_output_path your_output_bin_path
 ```
@@ -19,6 +22,7 @@ $ python tools/tokenizer.py --text_input_path your_input_text_path --bin_output_
 下面是一个数据处理的例子：
 
 给定一个包含原始数据集的文件`raw_data.txt`，原始数据集如下所示：
+
 ```bash
 感恩生活中的每一个细节，才能真正体会到幸福的滋味。
 梦想是人生的动力源泉，努力追逐，才能实现自己的目标。
@@ -26,6 +30,7 @@ $ python tools/tokenizer.py --text_input_path your_input_text_path --bin_output_
 ```
 
 可以通过运行以下命令来生成`bin`和`meta`文件：
+
 ```bash
 $ python tools/tokenizer.py --text_input_path raw_data.txt --bin_output_path cn/output.bin
 ```
@@ -35,19 +40,73 @@ $ python tools/tokenizer.py --text_input_path raw_data.txt --bin_output_path cn/
 其中，`cn`表示中文数据集；`en`表示英文数据集；`code`表示代码数据集；`ja`表示日语数据集；`ar`表示阿拉伯语数据集；`kaoshi`表示考试数据集。
 
 生成的bin文件的格式如下：
+
 ```python
 {"tokens": [73075, 75302, 69522, 69022, 98899, 67713, 68015, 81269, 74637, 75445, 99157]}
 {"tokens": [69469, 60355, 73026, 68524, 60846, 61844, 98899, 67775, 79241, 98899, 67713, 67800, 67453, 67838, 99157]}
 {"tokens": [68057, 79017, 60378, 68014, 98899, 67713, 67990, 68015, 70381, 67428, 61003, 67622, 99157]}
 ```
+
 `bin`文件中的每一行均对应原始数据集中的每一个句子，表示每个句子的`token`（下文将用sequence指定）。
 
 生成的`meta`文件的格式如下：
+
 ```bash
 (0, 11), (90, 15), (208, 13)
 ```
+
 在`meta`文件中，每个元组对应着`bin`文件中每一个`sequence`的元信息。其中，元组的第一个元素表示每个`sequence`在所有`sequence`中的`starting index`，第二个元素表示每个`sequence`中有多少个`tokens`。
 
 例如，对于第一个`sequence`，`starting index`为 0，有 11 个`tokens`；对于第二个`sequence`，由于第一个`sequence`转换为`string`后的长度为`89`，因此它的`starting index`为 90，有 15 个`tokens`。
 
-`json`和`jsonl`类型的文件的`bin`和`meta`文件格式和`txt`一致，此处不再赘叙。
\ No newline at end of file
+`json`和`jsonl`类型的文件的`bin`和`meta`文件格式和`txt`一致，此处不再赘叙。
+
+# pal_inference.py
+
+在 [GSM8K](https://huggingface.co/datasets/gsm8k) 数据集上使用 [PAL](https://github.com/reasoning-machines/pal) 范式推理，使模型编写代码并通过 Python 解释器执行来解决数学问题。其用法如下：
+
+```python
+# 用法:
+python pal_inference.py <model> <out_dir> [--dataset <dataset>] [--max_length <length>] [--top_p <threshold>] [--eoh <end token>] [--eoa <end token>] [--eos <end token>] [--temperature <temp>] [--time_out <time>] [--verbose, -v] [--append, -a]
+
+# 参数:
+# <model>                   用于推理的模型的路径。
+# <out_dir>                 生成代码将保存在指定的输出文件夹中。
+
+# 可选参数:
+# --dataset <dataset>       用于代码生成的数据集名称（默认：gsm8k）。
+# --max_length <length>     模型最大输入 token 长度（默认：2048）。
+# --top_p <threshold>       候选 token 相加的概率阈值（默认：0.8）。
+# --eoh <end token>         用户输入结束标识符 (默认: "") 。
+# --eoa <end token>         模型输入结束标识符 (默认: "") 。
+# --eos <end token>         系统输入结束标识符. (默认: "") 。
+# --temperature， -t <temp> 生成过程中的采样温度（默认：1.0）。
+# --time_out <time>         执行生成的代码的最大时间（秒）（默认：100）。
+# --verbose, -v             打印代码错误信息（可选）。
+# --append, -a              将输出追加到历史结果中（可选）。
+```
+
+以下是使用示例：
+
+```bash
+python tools/pal_inference.py internlm/internlm-chat-7k ./output -v
+```
+
+其输出文件每一行包括输入的问题，正确答案，执行答案，得分，以及模型生成的 Python 代码块：
+
+````json
+{
+    "question": "Janet\u2019s ducks lay 16 eggs per day. She eats three for breakfast every morning and bakes muffins for her friends every day with four. She sells the remainder at the farmers' market daily for $2 per fresh duck egg. How much in dollars does she make every day at the farmers' market?",
+    "target": 18.0,
+    "answer": 18.0,
+    "score": 1,
+    "generation": ["```python\ndef solution():\n    eggs_per_day = 16\n    eggs_per_breakfast = 3\n    eggs_per_muffin = 4\n    eggs_used = eggs_per_day - eggs_per_breakfast - eggs_per_muffin\n    eggs_sold = eggs_used\n    price_per_egg = 2\n    eggs_made = eggs_sold * price_per_egg\n    result = eggs_made\n    return result\n```"]
+}
+````
+
+InternLM 在 GSM8K 数据集中带工具和不带工具的性能表现：
+
+| Method   | **InternLM-Chat-7B** |
+| -------- | -------------------- |
+| w/o tool | 34.5                 |
+| w tool   | 39.2                 |
diff --git a/tools/README_EN.md b/tools/README_EN.md
index 140c36f..1385acd 100644
--- a/tools/README_EN.md
+++ b/tools/README_EN.md
@@ -1,4 +1,5 @@
 This directory provide some tools for model training with the following file structure.
+
 ```bash
 ├── transformers  # tools for adapting Hugging Face's transformers
 │   ├── configuration_internlm.py  # tools for adapting config
@@ -9,9 +10,11 @@ This directory provide some tools for model training with the following file str
 ```
 
 # tokenizer.py
+
 We need to use a `tokenizer` to generate `bin` and `meta` files for raw data. We import the tokenizer model by specifying the model weight path in `tools/tokenizer.py`. Currently, we provide `V7.model` to generate tokens. If you want to use a different model, you can modify the model weight path in `tokenizer.py` directly.
 
 We can run the following command to generate `bin` and `meta` files corresponding to the original data. The parameter `text_input_path` represents the path of the original text data, currently supporting `txt`, `json`, and `jsonl` formats, while `bin_output_path` represents the save path of the generated `bin` files.
+
 ```bash
 $ python tools/tokenizer.py --text_input_path your_input_text_path --bin_output_path your_output_bin_path
 ```
@@ -19,12 +22,15 @@ $ python tools/tokenizer.py --text_input_path your_input_text_path --bin_output_
 An example of data processing in `txt` format is given here:
 
 Given a file `raw_data.txt` containg raw data with the following content.
+
 ```bash
 Appreciate every detail in life to truly taste the flavor of happiness.
 Dreams are the source of life’s motivation. Pursue them diligently to achieve your goals.
 Learn to be tolerant and understanding to establish truly harmonious interpersonal relationships.
 ```
+
 Next, we can run the following command to generate `bin` and `meta` files for raw data.
+
 ```bash
 $ python tools/tokenizer.py --text_input_path your_input_text_path --bin_output_path your_output_bin_path
 ```
@@ -32,19 +38,73 @@ $ python tools/tokenizer.py --text_input_path your_input_text_path --bin_output_
 It should be noted that the generated `bin` files should be placed in one of the following directories to clarify the data type: `cn`(Chinese), `en`(English), `code`(code data), `ja`(Japanese), `ar`(Arabic) and `kaoshi`(kaoshi data).
 
 The format of generated `bin` file is as follows.
+
 ```python
 {"tokens": [98655, 2317, 2922, 6649, 1595, 7856, 435, 2424, 442, 9556, 12807, 410, 17313, 446, 23331, 95746]}
 {"tokens": [98655, 302, 1383, 269, 657, 410, 2687, 446, 2424, 98667, 269, 25220, 281, 523, 1874, 492, 1248, 38127, 4563, 442, 11227, 829, 8980, 95746]}
 {"tokens": [98655, 24190, 442, 517, 15013, 649, 454, 8793, 442, 5849, 9556, 17917, 1369, 1084, 29890, 12021, 95746]}
 ```
+
 In the generated `bin` file, each line (`sequence`) corresponds to the `tokens` for each sentence in the raw data.
 
 The format of generated `meta` file in as follows.
+
 ```bash
 (0, 16), (110, 24), (262, 17)
 ```
+
 Each tuple in the `meta` file represents the meta information of each `sequence` where the first element in the tuple indicates the `starting index` of each `sequence` among all `sequences` and the second element indicates the amount of `tokens` for each `sequence`.
 
 For example, the `starting index` is 0 for the first `sequence` with 16 `tokens`. Since the length of `sequence` in `string` format is 109, the `starting index` is 110. And the number of `tokens` of the sencond `sequence` is 24.
 
-The `bin` and `meta` file formats for `json` and `jsonl` type files are the same as for `txt`, so we won't go over them here.
\ No newline at end of file
+The `bin` and `meta` file formats for `json` and `jsonl` type files are the same as for `txt`, so we won't go over them here.
+
+# pal_inference.py
+
+Perform reasoning using [PAL](https://github.com/reasoning-machines/pal) on the [GSM8K](https://huggingface.co/datasets/gsm8k) dataset, allowing the model to generate code and solve mathematical problems through Python interpretation. Here's how you can use it:
+
+```bash
+# Usage:
+python pal_inference.py <model> <out_dir> [--dataset <dataset>] [--max_length <length>] [--top_p <threshold>] [--eoh <end token>] [--eoa <end token>] [--eos <end token>] [--temperature <temp>] [--time_out <time>] [--verbose, -v] [--append, -a]
+
+# Parameters:
+# <model>                   Path to the model used for inference.
+# <out_dir>                 Generated code will be saved in the specified output folder.
+
+# Optional arguments:
+# --dataset <dataset>       Dataset name used for code generation (default: gsm8k).
+# --max_length <length>     Model's maximum input token length (default: 2048).
+# --top_p <threshold>       Probability threshold for candidate tokens (default: 0.8).
+# --eoh <end token>         End of human (user) token. (default: "").
+# --eoa <end token>         End of assistant (bot) token. (default: "").
+# --eos <end token>         End of system token. (default: "").
+# --temperature, -t <temp>  Sampling temperature during generation (default: 1.0).
+# --time_out <time>         Maximum time (in seconds) for executing the generated code (default: 100).
+# --verbose, -v             Print code error messages (optional).
+# --append, -a              ppend the output to historical results (optional).
+```
+
+Below is an example of usage:
+
+```bash
+python tools/pal_inference.py internlm/internlm-chat-7k ./output -v
+```
+
+The output file contains each line with the input question, the correct answer, the executed answer, the score, and the Python code block generated by the model:
+
+````json
+{
+    "question": "Janet\u2019s ducks lay 16 eggs per day. She eats three for breakfast every morning and bakes muffins for her friends every day with four. She sells the remainder at the farmers' market daily for $2 per fresh duck egg. How much in dollars does she make every day at the farmers' market?",
+    "target": 18.0,
+    "answer": 18.0,
+    "score": 1,
+    "generation": ["```python\ndef solution():\n    eggs_per_day = 16\n    eggs_per_breakfast = 3\n    eggs_per_muffin = 4\n    eggs_used = eggs_per_day - eggs_per_breakfast - eggs_per_muffin\n    eggs_sold = eggs_used\n    price_per_egg = 2\n    eggs_made = eggs_sold * price_per_egg\n    result = eggs_made\n    return result\n```"]
+}
+````
+
+InternLM performance in the GSM8K dataset with and without tools:
+
+| Method   | **InternLM-Chat-7B** |
+| -------- | -------------------- |
+| w/o tool | 34.5                 |
+| w tool   | 39.2                 |
diff --git a/tools/pal_inference.py b/tools/pal_inference.py
new file mode 100644
index 0000000..703d169
--- /dev/null
+++ b/tools/pal_inference.py
@@ -0,0 +1,320 @@
+# This file is modified from:
+# hhttps://github.com/reasoning-machines/pal/blob/main/pal/core/interface.py
+#
+# Copyright 2022 PAL Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import argparse
+import copy
+import json
+import os
+from dataclasses import asdict
+from typing import Any, Dict, List
+
+import torch
+import tqdm
+from datasets import load_dataset
+from transformers import AutoModelForCausalLM, AutoTokenizer
+
+from internlm.utils.interface import GenerationConfig, generation_iterator
+from internlm.utils.timeout import Timeout
+
+
+def parse_args():
+    parser = argparse.ArgumentParser(description="PAL Inference")
+    parser.add_argument("model", type=str, help="Path to the pre-trained LLM used for inference.")
+    parser.add_argument(
+        "out_dir", type=str, help="Name of the output folder where generated code snippets will be saved."
+    )
+    parser.add_argument("--dataset", default="gsm8k", type=str, help="Name of the dataset used for code generation.")
+    parser.add_argument(
+        "--max_length",
+        default=2048,
+        type=int,
+        help="Maximum input token length for the natural language description.",
+    )
+    parser.add_argument(
+        "--top_p",
+        default=0.8,
+        type=float,
+        help="Probability threshold to choose sample tokens during generation.",
+    )
+    parser.add_argument(
+        "--eoh",
+        default="",
+        type=str,
+        help="End of human (user) token.",
+    )
+    parser.add_argument(
+        "--eoa",
+        default="",
+        type=str,
+        help="End of assistant (bot) token.",
+    )
+    parser.add_argument(
+        "--eos",
+        default="",
+        type=str,
+        help="End of system token.",
+    )
+    parser.add_argument(
+        "--temperature", "-t", default=1.0, type=float, help="Temperature of token sampling during generation."
+    )
+    parser.add_argument(
+        "--time_out", default=100, type=float, help="Maximum time allowed for executing generated code."
+    )
+    parser.add_argument(
+        "--verbose",
+        "-v",
+        action="store_true",
+        help="Print code error information when executing generated code (optional).",
+    )
+    parser.add_argument("--append", "-a", action="store_true", help="Append output to the history results (optional).")
+    args = parser.parse_args()
+    return args
+
+
+class GenericRuntime:
+    """Adapted from https://github.com/reasoning-machines/pal"""
+
+    GLOBAL_DICT: dict = {}
+    LOCAL_DICT = None
+    HEADERS: List = []
+
+    def __init__(self):
+        self._global_vars = copy.copy(self.GLOBAL_DICT)
+        self._local_vars = copy.copy(self.LOCAL_DICT) if self.LOCAL_DICT else None
+
+        for c in self.HEADERS:
+            self.exec_code(c)
+
+    def exec_code(self, code_piece: str) -> None:
+        exec(code_piece, self._global_vars)
+
+    def eval_code(self, expr: str) -> Any:
+        return eval(expr, self._global_vars)
+
+    def inject(self, var_dict: Dict[str, Any]) -> None:
+        for k, v in var_dict.items():
+            self._global_vars[k] = v
+
+    @property
+    def answer(self):
+        return self._global_vars["answer"]
+
+
+class PALInterface:
+    """PAL interface wrap fun:`generation_iterator` to extract and execute
+    generated code.
+
+    Adapted from https://github.com/reasoning-machines/pal
+
+    Args:
+        model (AutoModelForCausalLM)
+        tokenizer (AutoTokenizer)
+        generation_config (GenerationConfig): Decode strategies
+        additional_eos_token_id (int): End of sentence token id, default: 103028
+        get_answer_expr (str): The function name of generated code, default: "solution()"
+        verbose (bool): Print error information
+    """
+
+    def __init__(
+        self,
+        model: AutoModelForCausalLM,
+        tokenizer: AutoTokenizer,
+        generation_config: GenerationConfig,
+        additional_eos_token_id: int = 103028,
+        get_answer_expr: str = "solution()",
+        verbose: bool = False,
+    ):
+        self.runtime = GenericRuntime()
+        self.history: List = []
+        self.model = model
+        self.tokenizer = tokenizer
+        self.generation_config = generation_config
+        self.additional_eos_token_id = additional_eos_token_id
+        self.answer_expr = get_answer_expr
+        self.verbose = verbose
+
+    def generate(self, prompt):
+        # The api will generate response word by word
+        # we only need the last generation as the final results
+        for cur_gen in generation_iterator(
+            model=self.model,
+            tokenizer=self.tokenizer,
+            prompt=prompt,
+            additional_eos_token_id=self.additional_eos_token_id,
+            **asdict(self.generation_config),
+        ):
+            continue
+        # Get final response
+        self.history.append(cur_gen)
+        # Extract code block
+        code = self.process_generation_to_code(cur_gen)
+        return code
+
+    def process_generation_to_code(self, gens: str):
+        if "```python" in gens:
+            gens = gens.split("```python")[1].split("```")[0]
+        elif "```" in gens:
+            gens = gens.split("```")[1].split("```")[0]
+        code = gens.split("\n")
+        return code
+
+    def run(self, prompt, time_out: float = 100):
+        code = self.generate(prompt)
+        with Timeout(time_out):
+            try:
+                exec_result = self.execute(code)
+            except Exception as e:
+                if self.verbose:
+                    print(e)
+        return exec_result
+
+    def execute(self, code: List[str]):
+        self.runtime.exec_code("\n".join(code))
+        return self.runtime.eval_code(self.answer_expr)
+
+    def clear_history(self):
+        self.history = []
+
+
+def load_model(args):
+    model = AutoModelForCausalLM.from_pretrained(args.model, trust_remote_code=True).to(torch.bfloat16).cuda()
+    tokenizer = AutoTokenizer.from_pretrained(args.model, trust_remote_code=True)
+    return model, tokenizer
+
+
+def load_data(args):
+    # Load data from huggingface dataset
+    if args.dataset == "gsm8k":
+        gsm8k = load_dataset(path=args.dataset, name="main")
+        test_set = gsm8k["test"]
+        input_data = []
+        for data in test_set:
+            question = data["question"]
+            target = float(data["answer"].split("#")[-1].replace(",", ""))
+            input_data.append({"question": question, "target": target})
+    else:
+        raise NotImplementedError
+    return input_data
+
+
+PROMPT = """<|System|>:You are a helpful assistant which use tools to solve mathematical reasoning questions. The tools you can use are:
+PythonExecutor: It can execute Python code. The code must be a function, and the function name must be 'solution'. The example format is as follows:
+```python
+def solution():
+    variable_names_with_real_meaning = func(variable)
+    return variable_names_with_real_meaning
+```{eos}
+<|User|>:Olivia has $23. She bought five bagels for $3 each. How much money does she have left?{eoh}
+<|Bot|>:
+```python
+def solution():
+    money_initial = 23
+    bagels = 5
+    bagel_cost = 3
+    money_spent = bagels * bagel_cost
+    money_left = money_initial - money_spent
+    result = money_left
+    return result
+```{eoa}
+<|User|>:Michael had 58 golf balls. On tuesday, he lost 23 golf balls. On wednesday, he lost 2 more. How many golf balls did he have at the end of wednesday?{eoh}
+<|Bot|>:
+```python
+def solution():
+    golf_balls_initial = 58
+    golf_balls_lost_tuesday = 23
+    golf_balls_lost_wednesday = 2
+    golf_balls_left = golf_balls_initial - golf_balls_lost_tuesday - golf_balls_lost_wednesday
+    result = golf_balls_left
+    return result
+```{eoa}
+<|User|>:There were nine computers in the server room. Five more computers were installed each day, from monday to thursday. How many computers are now in the server room?{eoh}
+<|Bot|>:
+```python
+def solution():
+    computers_initial = 9
+    computers_per_day = 5
+    num_days = 4  # 4 days between monday and thursday
+    computers_added = computers_per_day * num_days
+    computers_total = computers_initial + computers_added
+    result = computers_total
+    return result
+```{eoa}
+<|System|>:How about this question?{eos}
+<|User|>:{question}{eoh}
+<|Bot|>:""".strip()
+
+
+def main():
+
+    args = parse_args()
+
+    print("load model begin.")
+    model, tokenizer = load_model(args)
+    print("load model end.")
+
+    generation_config = GenerationConfig(max_length=args.max_length, top_p=args.top_p, temperature=args.temperature)
+
+    verbose = args.verbose
+    interface = PALInterface(model=model, tokenizer=tokenizer, generation_config=generation_config, verbose=verbose)
+
+    if not os.path.exists(args.out_dir):
+        os.makedirs(args.out_dir)
+    savepath = os.path.join(args.out_dir, args.dataset + ".json")
+
+    # Load from history results
+    if args.append and os.path.exists(savepath):
+        lines = open(savepath).readlines()
+        num_skip_exps = len(lines)
+        scores = [x["score"] for x in map(json.loads, lines)]
+    else:
+        num_skip_exps = 0
+        scores = []
+
+    examples = load_data(args)
+    with open(savepath, "a" if args.append else "w") as f:
+        pbar = tqdm.tqdm(examples[num_skip_exps:], initial=num_skip_exps, total=len(examples))
+        for x in pbar:
+            question = x["question"]
+            result = copy.copy(x)
+
+            try:
+                answer = interface.run(
+                    prompt=PROMPT.format(question=question, eoh=args.eoh, eoa=args.eoa, eos=args.eos),
+                    time_out=args.time_out,
+                )
+                answer = float(answer)
+                score = 1 if abs(answer - x["target"]) < 1e-3 else 0
+            except Exception as e:
+                if verbose:
+                    print(e)
+                answer = ""
+                score = 0
+            scores.append(score)
+            result["answer"] = answer
+            result["score"] = score
+            result["generation"] = interface.history
+            f.write(json.dumps(result) + "\n")
+
+            interface.clear_history()
+            f.flush()
+
+    print(f"{args.model}: Accuracy - {sum(scores) / len(scores)}")
+    torch.cuda.empty_cache()
+
+
+if __name__ == "__main__":
+    main()