GPT-2模型
模型来源：从下述网站上下载config.json, merges.txt, tokenizer.json, vocal.json以及pytorch_model.bingpt2 at main (huggingfac
模型来源：
从下述网站上下载config.json, merges.txt, tokenizer.json, vocal.json以及pytorch_model.bin
gpt2 at main (huggingface.co)文件结构：
代码示例（finetune.py）：
from torch.utils.data import TensorDatasetnfrom torch.utils.data import DataLoadernfrom tqdm import trangennfrom transformers import GPT2Tokenizer, GPT2LMHeadModelnimport randomnimport torchnnnmodel_path = './models/gpt-2/'ndata_path = './data/'ntokenizer = GPT2Tokenizer.from_pretrained(model_path) n ndef select_top_k(predictions, k=10):n '''n 执行top-k选择，避免出现单词循环现象的出现n @params prediction: GPT-2模型对下一个词的预测向量n @params k : 默认 k=10n '''n predicted_index = random.choice(n predictions[0, -1, :].sort(descending=True)[1][:10]).item()n return predicted_indexnndef preprocess_data(path):n '''n 数据预处理n @params path: 数据的路径n return : List[int] 分成固定长度的"token索引片段"n '''n with open(path,'r') as f:n dataset = f.read()nn # print(len(dataset))nn indexed_text = tokenizer.encode(dataset)n del(dataset)nn dataset_cut = []n for i in range(len(indexed_text)//512):n # 将文本分段成 长度为512的片段n dataset_cut.append(indexed_text[i*512:(i+1)*512])n del(indexed_text)n return dataset_cutnndef do_train(epoch:int,optimizer:torch.optim.Optimizer,model,train_loader:DataLoader):n '''n 进行模型训练n @params epoch: 训练的循环次数n @params optimizer: 参数优化器n @params model: 使用的预训练模型(GPT-2)n @params train_loader: 训练数据集n '''n model.train()n for epoch in trange(epoch, desc="Epoch"):n total_loss = 0n for batch_idx ,(data,target) in enumerate(train_loader):n optimizer.zero_grad()n # print(batch_idx ,(data,target))n loss = model(data,labels=target).lossn total_loss += lossn loss.backward()n optimizer.step()n print('average loss:',total_loss/len(train_loader))nndef do_test(text,length,model):n '''n 模型根据前文内容预测后续句子n @params text: 前文n @params length: 后续句子的长度n @params model: 使用的预训练模型(GPT-2)n '''n indexed_tokens = tokenizer.encode(text)n tokens_tensor = torch.tensor([indexed_tokens])n n model.eval()n total_predicted_text = textn n # 使训练后的模型进行 500 次预测n for _ in range(length):n tokens_tensor = tokens_tensor.to('cuda')n n with torch.no_grad():n outputs = model(tokens_tensor)n predictions = outputs[0]n n predicted_index = select_top_k(predictions, k=10)n n predicted_text = tokenizer.decode(indexed_tokens + [predicted_index])n total_predicted_text += tokenizer.decode(predicted_index)n if '<|endoftext|>' in total_predicted_text:n # 如果出现文本结束标志，就结束文本生成n breakn n indexed_tokens += [predicted_index]n n if len(indexed_tokens) > 1023:n # 模型最长输入长度为1024，如果长度过长则截断n indexed_tokens = indexed_tokens[-1023:]n n tokens_tensor = torch.tensor([indexed_tokens])n n print(total_predicted_text)nndef main():n DEVICE = torch.device('cuda' if torch.cuda.is_available() else 'cpu')nn ## load datan dataset_cut = preprocess_data(data_path+'romeo_and_juliet.txt')n dataset_tensor = torch.tensor(dataset_cut).to(DEVICE)nn train_set = TensorDataset(dataset_tensor,dataset_tensor) # 标签与样本数据相同n train_loader = DataLoader(dataset = train_set,batch_size = 2, shuffle=False)nn ## load modeln model = GPT2LMHeadModel.from_pretrained(model_path+'pytorch_model.bin',config=model_path+'config.json')n model.to(DEVICE)nn epoch = 30n optimizer = torch.optim.Adam(model.parameters(),lr=1e-5)nn ## model trainingn do_train(epoch,optimizer,model,train_loader)nn ## do a testn do_test(text="At least, the whisper goes so.",length=500,model=model)n nn nif __name__ == "__main__":n main()
