# -*- coding: utf-8 -*- """ @Time : 2023/1/16 14:59 @Author : @FileName: @Software: @Describe: """ #! -*- coding: utf-8 -*- import os # os.environ["TF_KERAS"] = "1" os.environ["CUDA_VISIBLE_DEVICES"] = "1" import glob from numpy import random random.seed(1001) from tqdm import tqdm import numpy as np import pandas as pd import json import numpy as np from tqdm import tqdm from bert4keras.backend import keras, K from bert4keras.layers import Loss from bert4keras.models import build_transformer_model from bert4keras.tokenizers import SpTokenizer from bert4keras.optimizers import Adam from bert4keras.snippets import sequence_padding, open from bert4keras.snippets import DataGenerator, AutoRegressiveDecoder from keras.models import Model # from rouge import Rouge # pip install rouge # from nltk.translate.bleu_score import sentence_bleu, SmoothingFunction import tensorflow as tf from keras.backend import set_session config = tf.ConfigProto() config.gpu_options.allow_growth = True set_session(tf.Session(config=config)) # 此处不同 global graph graph = tf.get_default_graph() sess = tf.Session(graph=graph) set_session(sess) # global graph,model # graph = tf.get_default_graph() # sess = tf.Session(graph=graph) # K.set_session(sess) # 基本参数 class GenerateModel(object): def __init__(self): self.epoch_acc_vel = 0 self.config_path = 'mt5/mt5_base/mt5_base_config.json' self.checkpoint_path = 'mt5/mt5_base/model.ckpt-1000000' self.spm_path = 'mt5/mt5_base/sentencepiece_cn.model' self.keep_tokens_path = 'mt5/mt5_base/sentencepiece_cn_keep_tokens.json' self.maxlen = 256 def device_setup(self): tokenizer = SpTokenizer(self.spm_path, token_start=None, token_end='') keep_tokens = json.load(open(self.keep_tokens_path)) t5 = build_transformer_model( config_path=self.config_path, checkpoint_path=self.checkpoint_path, keep_tokens=keep_tokens, model='mt5.1.1', return_keras_model=False, name='T5', ) # output = CrossEntropy(2)(model.inputs + model.outputs) # # model = Model(model.inputs, output) encoder = t5.encoder decoder = t5.decoder model = t5.model model.summary() output = CrossEntropy(1)([model.inputs[1], model.outputs[0]]) model = Model(model.inputs, output) path_model = "output_t5/best_model_t5.weights" model.load_weights(path_model) return encoder, decoder, model, tokenizer class CrossEntropy(Loss): """交叉熵作为loss，并mask掉输入部分 """ def compute_loss(self, inputs, mask=None): y_true, y_pred = inputs y_true = y_true[:, 1:] # 目标token_ids y_mask = K.cast(mask[1], K.floatx())[:, 1:] # 解码器自带mask y_pred = y_pred[:, :-1] # 预测序列，错开一位 loss = K.sparse_categorical_crossentropy(y_true, y_pred) loss = K.sum(loss * y_mask) / K.sum(y_mask) return loss class Beamdataone(object): def __init__(self, num_beams, batch_id, text, end_id, minlen, min_ends, tokenizer, output_ids): """ Initialize n-best list of hypotheses. """ self.num_beams = num_beams self.batch_id = batch_id self.beams = [] self.minlen = minlen self.min_ends = min_ends self.end_id = end_id self.text = text self.output_scores = np.zeros(1) self.output_ids = [output_ids] self.return_str = "" self.over = False self.tokenizer = tokenizer # self.data() self.output_str = "" self.text_2_textids( self.text ) self.scores = np.zeros(1) self.inputs_vector = 0 def text_2_textids(self,text): token_ids, segment_ids = self.tokenizer.encode(text[0], maxlen=120) self.text_ids = [token_ids] def add_data(self, step, output_scores): ''' 还存有的数据，直接可以被迭代， @param text: @return: ''' # inputs = [np.array([i]) for i in inputs] # output_ids, output_scores = self.first_output_ids, np.zeros(1) # # scores, states = self.predict( # inputs, output_ids, states, temperature, 'logits' # ) # 计算当前得分 # if step == 0: # 第1步预测后将输入重复topk次 # inputs = [np.repeat(i, self.num_beams, axis=0) for i in self.inputs] # inputs = [self.token_ids, self.segment_ids] # inputs = [np.array([i]) for i in inputs] self.output_ids = np.array(self.output_ids) if step == 0: # 第1步预测后将输入重复topk次 self.text_ids = [np.repeat(i, self.num_beams, axis=0) for i in self.text_ids] scores = output_scores.reshape((-1, 1)) + self.scores # 综合累积得分 # scores = output_probas scores = self.output_scores.reshape((-1, 1)) + scores # 综合累积得分 indices = scores.argpartition(-self.num_beams, axis=None)[-self.num_beams:] # 仅保留topk indices_1 = indices // scores.shape[1] # 行索引 indices_2 = (indices % scores.shape[1]).reshape((-1, 1)) # 列索引 self.output_ids = np.concatenate([self.output_ids[indices_1], indices_2], 1) # 更新输出 self.output_scores = np.take_along_axis( scores, indices, axis=None ) # 更新得分 is_end = self.output_ids[:, -1] == self.end_id # 标记是否以end标记结束 self.end_counts = (self.output_ids == self.end_id).sum(1) # 统计出现的end标记 if self.output_ids.shape[1] >= self.minlen: # 最短长度判断 best = self.output_scores.argmax() # 得分最大的那个 if is_end[best] and self.end_counts[best] >= self.min_ends: # 如果已经终止 # return output_ids[best] # 直接输出 self.return_str_main(self.output_ids, best) self.over = True else: # 否则，只保留未完成部分 flag = ~is_end | (self.end_counts < self.min_ends) # 标记未完成序列 if not flag.all(): # 如果有已完成的 self.output_ids = self.output_ids[flag] # 扔掉已完成序列 self.output_scores = self.output_scores[flag] # 扔掉已完成序列 self.end_counts = self.end_counts[flag] # 扔掉已完成end计数 self.num_beams = flag.sum() # topk相应变化 self.output_ids = self.output_ids.tolist() self.output_str = [tokenizer.decode(ids) for ids in self.output_ids] self.text_ids = [self.text_ids[0] for i in range(len(self.output_ids))] # # 达到长度直接输出 # return output_ids[output_scores.argmax()] # def data(self): # token_ids, segment_ids = self.tokenizer.encode(self.text, maxlen=256) # self.token_ids = token_ids # self.segment_ids = segment_ids # input_str = [text for i in range(self.num_beams)] # output_str = self.output_str # return input_str, output_str def return_str_main(self, output_ids, best): output_ids_best = output_ids[best] self.return_str = self.tokenizer.decode(output_ids_best) class AutoTitle(AutoRegressiveDecoder): """seq2seq解码器 """ def __init__(self, encoder, decoder, model, tokenizer, start_id, end_id, maxlen, minlen=1): super(AutoTitle, self).__init__(start_id, end_id, maxlen, minlen) self.encoder = encoder self.decoder = decoder self.model = model self.tokenizer = tokenizer self.start_id = start_id self.end_id = end_id self.minlen = minlen self.models = {} if start_id is None: self.first_output_ids = np.empty((1, 0), dtype=int) else: self.first_output_ids = np.array([[self.start_id]]) # @AutoRegressiveDecoder.wraps(default_rtype='probas') # def predict(self, inputs, output_ids, states): # token_ids, segment_ids = inputs # token_ids = np.concatenate([token_ids, output_ids], 1) # segment_ids = np.concatenate([segment_ids, np.ones_like(output_ids)], 1) # with graph.as_default(): # K.set_session(sess) # nodes = self.last_token(self.model).predict([token_ids, segment_ids]) # return nodes # # return self.last_token(self.model).predict([token_ids, segment_ids]) # @AutoRegressiveDecoder.wraps(default_rtype='probas') # def predict(self, inputs, output_ids, states): # c_encoded = inputs[0] # with graph.as_default(): # K.set_session(sess) # nodes = self.last_token(self.decoder).predict([c_encoded, output_ids]) # return nodes @AutoRegressiveDecoder.wraps(default_rtype='probas') def predict(self, inputs, output_ids, states): c_encoded = inputs[0] with graph.as_default(): K.set_session(sess) nodes = self.last_token(decoder).predict([c_encoded, output_ids]) return nodes def predict_batch(self, inputs): # inputs, output_ids, states, temperature, 'probas' token_ids, output_ids = inputs # token_ids = np.concatenate([token_ids, output_ids], 1) # segment_ids = np.concatenate([segment_ids, np.ones_like(output_ids)], 1) with graph.as_default(): K.set_session(sess) nodes = self.decoder.predict([token_ids, output_ids]) return nodes def data_generator(self, token_ids, output_ids): batch_token_ids = [] for i,j in zip(token_ids, output_ids): batch_token_ids = sequence_padding(token_ids) batch_segment_ids = sequence_padding(output_ids) return batch_token_ids, batch_segment_ids def beam_search_batch( self, inputs_str, states=None, temperature=1, min_ends=1, num_beam=3 ): """随机采样n个结果说明：非None的topk表示每一步只从概率最高的topk个中采样；而非None的topp 表示每一步只从概率最高的且概率之和刚好达到topp的若干个token中采样。返回：n个解码序列组成的list。 """ output_str = [] # token_ids, segment_ids = self.data_generator(inputs, output_ids) batch_nums = len(inputs_str) return_str_batch = [0] * batch_nums # output_ids = np.empty((batch_nums, 0), dtype=int) output_ids = np.array([self.start_id]) generated = [Beamdataone(num_beam, i, [inputs_str[i]], self.end_id, self.minlen, min_ends, self.tokenizer, output_ids) for i in range(batch_nums)] # index_data = [i for i in range(batch_nums)] c_token_ids = [] for i in generated: text_ids = i.text_ids c_token_ids.extend(text_ids) c_token_ids = sequence_padding(c_token_ids) c_encoded = encoder.predict(np.array(c_token_ids)) # probas_bool = np.array(token_ids, dtype=bool) # # np.array(np.where(probas_bool == True)) # for i, sentence in enumerate(probas_bool): # lie = np.array(np.where(sentence == True))[0] # probas_new.append(probas[i, lie[-1]]) for i in range(len(generated)): probas_bool = np.array(generated[i].text_ids[0], dtype=bool) lie = np.array(np.where(probas_bool == True))[0] # c_encoded_dan = c_encoded[i, lie[-1]] c_encoded_dan = c_encoded[np.ix_([i], lie)] generated[i].inputs_vector = c_encoded_dan[0] for step in range(self.maxlen): # if step == 0: # token_ids, segment_ids = self.data_generator(inputs_str, output_str) # else: # inputs_str, output_str = [], [] inputs_vector_batch, output_ids_batch = [], [] batch_input_num_beam_num = [] for i in generated: inputs_vector = i.inputs_vector # if step != 0: # output_ids_batch.extend(i.output_ids) # text_ids_batch.extend(text_ids) # else: inputs_vector_batch.append(inputs_vector) output_ids_batch.extend(i.output_ids) if step != 0: batch_input_num_beam_num.append(i.num_beams) # token_ids, output_ids_batch = self.data_generator(inputs_vector_batch, output_ids_batch) # token_ids_batch = sequence_padding(token_ids_batch) # segment_ids_batch = sequence_padding(segment_ids_batch) # output_ids_batch = np.array(output_ids_batch) # if step == 0: inputs = [inputs_vector_batch, output_ids_batch] probas = self.predict_batch( inputs ) # 计算当前概率 probas_new = [] probas_bool = np.array(inputs_vector_batch, dtype=bool) # np.array(np.where(probas_bool == True)) for i, sentence in enumerate(probas_bool): lie = np.array(np.where(sentence == True))[0] probas_new.append(probas[i, lie[-1]]) probas = np.array(probas_new) if step != 0: num = 0 if len(generated) > 1: index = 0 for index in range(len(batch_input_num_beam_num)-1): cc = num num += batch_input_num_beam_num[index] generated[index].add_data(step, probas[cc:num,:]) generated[index+1].add_data(step, probas[num:,:]) else: generated[0].add_data(step, probas[:,:]) else: for index in range(len(generated)): generated[index].add_data(step, probas[index,:]) # i = 0 # while True: # bool_ = generated[i].over # if bool_ == True: # one_sentence = generated.pop(i) # return_str_batch[i] = one_sentence.return_str # if i > len(generated) - 1: # break # else: # i += 1 # if i > len(generated) - 1: # break generated_new = [] for i in range(len(generated)): bool_ = generated[i].over if bool_ == False: generated_new.append(generated[i]) else: return_str_batch[generated[i].batch_id] = generated[i].return_str generated = generated_new if generated == []: return return_str_batch return return_str_batch def generate(self, text, topk=5): c_token_ids, _ = tokenizer.encode(text, maxlen=120) with graph.as_default(): K.set_session(sess) c_encoded = encoder.predict(np.array([c_token_ids]))[0] output_ids = self.beam_search([c_encoded], topk=topk) # 基于beam search return tokenizer.decode([int(i) for i in output_ids]) def generate_random(self, text, n=30, topp=0.9): c_token_ids, _ = self.tokenizer.encode(text, maxlen=120) with graph.as_default(): K.set_session(sess) c_encoded = self.encoder.predict(np.array([c_token_ids]))[0] output_ids = self.random_sample([c_encoded], n, topp=topp) # 基于随机采样 text = [] for ids in output_ids: text.append(tokenizer.decode([int(i) for i in ids])) return text def generate_beam_search_batch(self, text): output_str = self.beam_search_batch(text) # 基于随机采样 return output_str generatemodel = GenerateModel() encoder, decoder, model, tokenizer = generatemodel.device_setup() autotitle = AutoTitle(encoder, decoder, model, tokenizer, start_id=0, end_id=tokenizer._token_end_id, maxlen=120) def just_show_sentence(file): """ @param file:list """ text = file[0] pre = autotitle.generate(text) return pre def just_show_sentence_batch(file: list) -> object: text = file pre = autotitle.generate_beam_search_batch(text) return pre if __name__ == '__main__': # file = "train_2842.txt" # just_show(file) # text = ["历史和当下都证明,创新是民族生存、发展的不竭源泉,是自身发展的必然选择,是时代对于青年们的深切呼唤"] # a = just_show_sentence(text) # print(a) # print(type(a)) # ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ # is_novel = False # path = "./data/700条论文测试.xlsx" # df_list = pd.read_excel(path).values.tolist() # # # df_list_new = [] # print(len(df_list)) # for i in tqdm(df_list): # pre = just_show_sentence([i[0]]) # # df_list_new.append([i[0], i[1], pre]) # # df = pd.DataFrame(df_list_new, columns=["原文", "yy降重", "t5模型"]) # df.to_excel("./data/700条论文测试_7.xlsx", index=None) # ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ # import os # # file = "./data/11篇汇总txt_new.txt" # file_t5 = "./data/11篇汇总txt_new_predict_t5.txt" # # try: # with open(file, 'r', encoding="utf-8") as f: # lines = [x.strip() for x in f if x.strip() != ''] # except: # with open(file, 'r', encoding="gbk") as f: # lines = [x.strip() for x in f if x.strip() != ''] # # zishu = 0 # data = [] # for i in tqdm(lines): # # zishu += len(i) # pre = just_show_sentence([i]) # data.append([i, pre]) # # with open(file_t5, "w", encoding='utf-8') as file: # for i in data: # file.write("\t".join(i) + '\n') # file.close() # print(zishu) #++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ text = ["'李正旺你真是傻逼讪笑，挥手道：“不不不，你千万别误会", "历史和当下都证明,创新是民族生存、“发展的不竭源泉”,是是自身发展的必然选择", "自身发展的必然选择", "强调轻资产经营, 更加重视经营风险的规避", "历史和当下都证明,创新是民族生存、发展的不竭源泉,是是自身发展的必然选择", "是时代对于青年们的深切呼唤"] # text = ["基本消除“热桥”影响。"] print(just_show_sentence(text)) # print(just_show_sentence_top(text)) # print(just_show_chachong_random(text)) # print(tokenizer.encode("\"", maxlen=120)) # print(just_show_sentence_batch(text))