mirror of https://github.com/THUDM/ChatGLM2-6B
finetune t5
wanghh2000
parent
d4048deebf
commit
78abaa4b71
|
|
@ -0,0 +1,78 @@
|
|||
from transformers import pipeline, AutoTokenizer, AutoModelForSeq2SeqLM, Seq2SeqTrainer, Seq2SeqTrainingArguments, DataCollatorForSeq2Seq
|
||||
|
||||
import torch
|
||||
device = torch.device("cpu")
|
||||
|
||||
checkpoint = "/Users/hhwang/models/t5-small"
|
||||
# checkpoint = "/Users/hhwang/models/flan-t5-small"
|
||||
|
||||
print('********* before finetune ***********')
|
||||
tokenizer = AutoTokenizer.from_pretrained(checkpoint,use_fast=False)
|
||||
model = AutoModelForSeq2SeqLM.from_pretrained(checkpoint)
|
||||
# print(model.config)
|
||||
inputs = tokenizer.encode("translate English to Chinese: That is good", return_tensors="pt")
|
||||
outputs = model.generate(inputs, max_new_tokens=20)
|
||||
print('result: ',tokenizer.batch_decode(outputs))
|
||||
|
||||
data = [
|
||||
{"question": "今天天真好", "answer": "那一起打篮球去吧"},
|
||||
{"question": "translate English to Chinese: That is good", "answer": "Not bad"}
|
||||
]
|
||||
|
||||
def preprocess_function(examples):
|
||||
inputs = tokenizer(examples["question"], max_length=32, truncation=True)
|
||||
labels = tokenizer(examples["answer"], max_length=32, truncation=True)
|
||||
inputs["labels"] = labels["input_ids"]
|
||||
return inputs
|
||||
|
||||
from datasets import Dataset, load_dataset
|
||||
dataset = Dataset.from_list(data)
|
||||
dataset = dataset.map(preprocess_function, batched=True, remove_columns=dataset.column_names)
|
||||
print(dataset)
|
||||
|
||||
data_collator = DataCollatorForSeq2Seq(tokenizer=tokenizer, model=model)
|
||||
# print(data_collator([dataset[0], dataset[1]]))
|
||||
|
||||
training_args = Seq2SeqTrainingArguments(
|
||||
output_dir="checkpoints",
|
||||
overwrite_output_dir=True,
|
||||
use_cpu=True,
|
||||
do_train=True,
|
||||
do_eval=True,
|
||||
learning_rate=1e-3,
|
||||
lr_scheduler_type="constant",
|
||||
per_device_train_batch_size=16,
|
||||
per_device_eval_batch_size=16,
|
||||
num_train_epochs=10,
|
||||
weight_decay=0.01,
|
||||
save_steps=10,
|
||||
save_total_limit=5,
|
||||
logging_first_step=True,
|
||||
logging_steps=1,
|
||||
# logging_dir="./",
|
||||
eval_steps=1,
|
||||
evaluation_strategy="steps",
|
||||
load_best_model_at_end=True
|
||||
)
|
||||
|
||||
trainer = Seq2SeqTrainer(
|
||||
model=model,
|
||||
args=training_args,
|
||||
train_dataset=dataset,
|
||||
eval_dataset=dataset,
|
||||
data_collator=data_collator,
|
||||
# compute_metrics=compute_metrics
|
||||
)
|
||||
|
||||
print('begin train')
|
||||
trainer.train()
|
||||
print('done train')
|
||||
|
||||
finetune_mode = "/tmp/outputs/t5-small"
|
||||
trainer.save_model(finetune_mode)
|
||||
|
||||
print('********* after finetune ***********')
|
||||
prompt = "translate English to Chinese: That is good"
|
||||
model = AutoModelForSeq2SeqLM.from_pretrained(finetune_mode)
|
||||
generator = pipeline("summarization", model=model, tokenizer=tokenizer)
|
||||
print(generator(prompt))
|
||||
|
|
@ -1,75 +0,0 @@
|
|||
|
||||
from transformers import AutoModelForSeq2SeqLM, AutoTokenizer, AutoModel, AutoModelForCausalLM
|
||||
from transformers import pipeline
|
||||
|
||||
checkpoint = "bigscience/mt0-large"
|
||||
checkpoint = "/Users/hhwang/models/gpt2"
|
||||
checkpoint = "/Users/hhwang/models/opt-125m"
|
||||
checkpoint = "/Users/hhwang/models/opt-350m"
|
||||
|
||||
# tokenizer = AutoTokenizer.from_pretrained(checkpoint)
|
||||
# model = AutoModelForSeq2SeqLM.from_pretrained(checkpoint)
|
||||
# inputs = tokenizer.encode("Write a short story", return_tensors="pt")
|
||||
# outputs = model.generate(inputs)
|
||||
# print(tokenizer.decode(outputs[0]))
|
||||
|
||||
# case 1
|
||||
# pipe = pipeline(task='text-generation', model=checkpoint)
|
||||
# print(pipe)
|
||||
# result = pipe("tell me a joke")
|
||||
# print('result: ',result)
|
||||
|
||||
# case 2
|
||||
# from transformers import GPT2Tokenizer, GPT2LMHeadModel
|
||||
# tokenizer = GPT2Tokenizer.from_pretrained(checkpoint)
|
||||
# model = GPT2LMHeadModel.from_pretrained(checkpoint)
|
||||
# text = "Replace me by any text you'd like."
|
||||
# encoded_input = tokenizer.encode(text, return_tensors='pt')
|
||||
# outputs = model.generate(encoded_input, max_length=50, num_return_sequences=1)
|
||||
# generated_texts = tokenizer.batch_decode(outputs, skip_special_tokens=True)
|
||||
# for i, generated_text in enumerate(generated_texts):
|
||||
# print(f"Generated text {i + 1}: {generated_text}")
|
||||
|
||||
# # case 3
|
||||
# from transformers import GPT2Tokenizer, GPT2Model
|
||||
# tokenizer = GPT2Tokenizer.from_pretrained(checkpoint)
|
||||
# model = GPT2Model.from_pretrained(checkpoint)
|
||||
# text = "Replace me by any text you'd like."
|
||||
# encoded_input = tokenizer(text, return_tensors='pt')
|
||||
# outputs = model(**encoded_input)
|
||||
# print(outputs)
|
||||
# last_hidden_states = outputs.last_hidden_state
|
||||
# print(last_hidden_states)
|
||||
|
||||
# case 4
|
||||
# tokenizer = AutoTokenizer.from_pretrained(checkpoint)
|
||||
# model = AutoModel.from_pretrained(checkpoint)
|
||||
# inputs = tokenizer.encode("Write a short story", return_tensors="pt")
|
||||
# model = model.eval()
|
||||
# print(inputs)
|
||||
# outputs = model(inputs)
|
||||
# print(outputs)
|
||||
|
||||
# case 5
|
||||
print('********* case 5 ***********')
|
||||
tokenizer = AutoTokenizer.from_pretrained(checkpoint)
|
||||
model = AutoModelForCausalLM.from_pretrained(checkpoint)
|
||||
inputs = tokenizer.encode("Write a short story", return_tensors="pt")
|
||||
outputs = model.generate(inputs)
|
||||
print('result: ',tokenizer.batch_decode(outputs))
|
||||
|
||||
# case 6
|
||||
print('********* case 6 ***********')
|
||||
from transformers import GPT2Tokenizer, OPTForCausalLM
|
||||
model = OPTForCausalLM.from_pretrained(checkpoint)
|
||||
tokenizer = GPT2Tokenizer.from_pretrained(checkpoint)
|
||||
prompt = "Anti Vaccine Movemenet"
|
||||
inputs = tokenizer(prompt, return_tensors="pt").input_ids
|
||||
|
||||
gen_tokens = model.generate(inputs,do_sample=True,temperature=0.9,max_length=100)
|
||||
gen_text = tokenizer.batch_decode(gen_tokens)[0]
|
||||
print('gen_text', gen_text)
|
||||
# generate_ids = model.generate(inputs,max_length=2000,early_stopping= True,do_sample=True,min_length=2000,top_k=125,top_p=0.92,temperature= 0.85,repetition_penalty=1.5,num_return_sequences=3)
|
||||
# for i, sample_output in enumerate(generate_ids):
|
||||
# result = tokenizer.decode(sample_output, skip_special_tokens=True, clean_up_tokenization_spaces=True)
|
||||
# print(result)
|
||||
|
|
@ -0,0 +1,138 @@
|
|||
|
||||
from transformers import AutoModelForSeq2SeqLM, AutoTokenizer, AutoModel, AutoModelForCausalLM
|
||||
from transformers import pipeline
|
||||
|
||||
import torch
|
||||
device = torch.device("cpu")
|
||||
|
||||
checkpoint = "bigscience/mt0-large"
|
||||
checkpoint = "/Users/hhwang/models/gpt2"
|
||||
# checkpoint = "/Users/hhwang/models/opt-125m"
|
||||
# checkpoint = "/Users/hhwang/models/opt-350m"
|
||||
|
||||
# tokenizer = AutoTokenizer.from_pretrained(checkpoint)
|
||||
# model = AutoModelForSeq2SeqLM.from_pretrained(checkpoint)
|
||||
# inputs = tokenizer.encode("Write a short story", return_tensors="pt")
|
||||
# outputs = model.generate(inputs)
|
||||
# print(tokenizer.decode(outputs[0]))
|
||||
|
||||
# case 1
|
||||
# pipe = pipeline(task='text-generation', model=checkpoint)
|
||||
# print(pipe)
|
||||
# result = pipe("tell me a joke")
|
||||
# print('result: ',result)
|
||||
|
||||
# case 2
|
||||
print('********* case 2 ***********')
|
||||
from transformers import GPT2Tokenizer, GPT2LMHeadModel
|
||||
tokenizer = GPT2Tokenizer.from_pretrained(checkpoint)
|
||||
model = GPT2LMHeadModel.from_pretrained(checkpoint)
|
||||
text = "Replace me by any text you'd like."
|
||||
encoded_input = tokenizer.encode(text, return_tensors='pt')
|
||||
outputs = model.generate(encoded_input, max_length=50, num_return_sequences=1)
|
||||
print('outputs:', outputs)
|
||||
print(outputs.shape)
|
||||
generated_texts = tokenizer.batch_decode(outputs, skip_special_tokens=True)
|
||||
for i, generated_text in enumerate(generated_texts):
|
||||
print(f"Generated text {i + 1}: {generated_text}")
|
||||
|
||||
# case 3
|
||||
print('********* case 3 ***********')
|
||||
from transformers import GPT2Tokenizer, GPT2Model
|
||||
tokenizer = GPT2Tokenizer.from_pretrained(checkpoint)
|
||||
model = GPT2Model.from_pretrained(checkpoint)
|
||||
print('config', model.config)
|
||||
# print('model', model)
|
||||
text = "Replace me by any text you'd like."
|
||||
encoded_input = tokenizer(text, return_tensors='pt')
|
||||
outputs = model(**encoded_input)
|
||||
# print(outputs)
|
||||
last_hidden_states = outputs.last_hidden_state
|
||||
print('last_hidden_states', last_hidden_states)
|
||||
print(last_hidden_states.shape)
|
||||
print(len(last_hidden_states[0][0]))
|
||||
import torch.nn as nn
|
||||
lm_head = nn.Linear(model.config.n_embd, model.config.vocab_size, bias=False)
|
||||
lm_logits = lm_head(last_hidden_states)
|
||||
print('lm_logits', lm_logits)
|
||||
print(lm_logits.shape)
|
||||
|
||||
# case 4
|
||||
# print('********* case 4 ***********')
|
||||
# tokenizer = AutoTokenizer.from_pretrained(checkpoint)
|
||||
# model = AutoModel.from_pretrained(checkpoint)
|
||||
# encoded_input = tokenizer.encode("Write a short story", return_tensors="pt")
|
||||
# model = model.eval()
|
||||
# print('config', model.config)
|
||||
# print('model', model)
|
||||
# print('inputs', encoded_input)
|
||||
# outputs = model(encoded_input)
|
||||
# print(outputs)
|
||||
|
||||
# case 5
|
||||
# print('********* case 5 ***********')
|
||||
# tokenizer = AutoTokenizer.from_pretrained(checkpoint)
|
||||
# model = AutoModelForCausalLM.from_pretrained(checkpoint)
|
||||
# inputs = tokenizer.encode("Write a short story", return_tensors="pt")
|
||||
# outputs = model.generate(inputs)
|
||||
# print('result: ',tokenizer.batch_decode(outputs))
|
||||
|
||||
# case 6
|
||||
# print('********* case 6 ***********')
|
||||
# from transformers import GPT2Tokenizer, OPTForCausalLM
|
||||
# model = OPTForCausalLM.from_pretrained(checkpoint)
|
||||
# tokenizer = GPT2Tokenizer.from_pretrained(checkpoint)
|
||||
# prompt = "Anti Vaccine Movemenet"
|
||||
# inputs = tokenizer(prompt, return_tensors="pt").input_ids
|
||||
|
||||
# gen_tokens = model.generate(inputs,do_sample=True,temperature=0.9,max_length=100)
|
||||
# gen_text = tokenizer.batch_decode(gen_tokens)[0]
|
||||
# print('gen_text', gen_text)
|
||||
# generate_ids = model.generate(inputs,max_length=2000,early_stopping= True,do_sample=True,min_length=2000,top_k=125,top_p=0.92,temperature= 0.85,repetition_penalty=1.5,num_return_sequences=3)
|
||||
# for i, sample_output in enumerate(generate_ids):
|
||||
# result = tokenizer.decode(sample_output, skip_special_tokens=True, clean_up_tokenization_spaces=True)
|
||||
# print(result)
|
||||
|
||||
# case 7
|
||||
# print('********* case 7 ***********')
|
||||
# generator = pipeline('text-generation', model=checkpoint, device="cpu")
|
||||
# text_inputs = ["tell me joke", "How do you", "Would you help", "I like apple", "This is something"]
|
||||
# sent_gen = generator(text_inputs, max_length=50, num_return_sequences=2, repetition_penalty=1.3, top_k = 20)
|
||||
# #返回的sent_gen 形如#[[{'generated_text':"..."},{}],[{},{}]]
|
||||
# for i in sent_gen:
|
||||
# print(i)
|
||||
|
||||
# case 8
|
||||
# print('********* case 8 ***********')
|
||||
# from transformers import GPT2Tokenizer, GPT2LMHeadModel, TextGenerationPipeline
|
||||
# tokenizer = GPT2Tokenizer.from_pretrained(checkpoint)
|
||||
# model = GPT2LMHeadModel.from_pretrained(checkpoint)
|
||||
# text_generator = TextGenerationPipeline(model, tokenizer, batch_size=3, device="cpu")
|
||||
# text_generator.tokenizer.pad_token_id = model.config.eos_token_id
|
||||
# text_inputs = ["tell me joke", "How do you", "Would you help", "I like apple", "This is something"]
|
||||
# gen = text_generator(text_inputs, max_length=50, repetition_penalty=10.0, do_sample=True, num_beams=5, top_k=10)
|
||||
# for sent in gen:
|
||||
# gen_seq = sent[0]["generated_text"]
|
||||
# print("")
|
||||
# print(gen_seq)
|
||||
|
||||
# case 9
|
||||
# print('********* case 9 ***********')
|
||||
# from transformers import AutoTokenizer, AutoModelWithLMHead
|
||||
# import torch
|
||||
# tokenizer = AutoTokenizer.from_pretrained(checkpoint)
|
||||
# model = AutoModelWithLMHead.from_pretrained(checkpoint)
|
||||
# config=model.config
|
||||
# # print('config', config)
|
||||
# print(model)
|
||||
# device = 'cuda' if torch.cuda.is_available() else 'cpu'
|
||||
# model = model.to(device)
|
||||
# texts = ["tell me joke", "How do you", "Would you help", "I like apple", "This is something"]
|
||||
# #用batch输入的时候一定要设置padding
|
||||
# tokenizer.pad_token = tokenizer.eos_token
|
||||
# encoding = tokenizer(texts, return_tensors='pt', padding=True).to(device)
|
||||
# with torch.no_grad():
|
||||
# generated_ids = model.generate(**encoding, max_length=50, do_sample=True, top_k=20, repetition_penalty=3.0)
|
||||
# generated_texts = tokenizer.batch_decode(generated_ids, skip_special_tokens=True)
|
||||
# for l in generated_texts:
|
||||
# print(l)
|
||||
|
|
@ -0,0 +1,24 @@
|
|||
from transformers import AutoTokenizer, AutoModelForSeq2SeqLM
|
||||
|
||||
import torch
|
||||
device = torch.device("cpu")
|
||||
|
||||
checkpoint = "/Users/hhwang/models/t5-small"
|
||||
checkpoint = "/Users/hhwang/models/flan-t5-small"
|
||||
|
||||
print('********* case 1 ***********')
|
||||
# tokenizer = AutoTokenizer.from_pretrained(checkpoint)
|
||||
# model = AutoModelForSeq2SeqLM.from_pretrained(checkpoint)
|
||||
# # print(model.config)
|
||||
# inputs = tokenizer.encode("translate English to German: That is good", return_tensors="pt")
|
||||
# outputs = model.generate(inputs, max_new_tokens=20)
|
||||
# print('result: ',tokenizer.batch_decode(outputs))
|
||||
|
||||
print('********* case 2 ***********')
|
||||
|
||||
from transformers import pipeline
|
||||
tokenizer = AutoTokenizer.from_pretrained(checkpoint)
|
||||
model = AutoModelForSeq2SeqLM.from_pretrained(checkpoint)
|
||||
prompt = "translate English to German: That is good?"
|
||||
generator = pipeline("summarization", model=model, tokenizer=tokenizer)
|
||||
print(generator(prompt))
|
||||
Loading…
Reference in New Issue