You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
1109 lines
44 KiB
1109 lines
44 KiB
#! -*- coding: utf-8 -*-
|
|
|
|
import os
|
|
# os.environ["TF_KERAS"] = "1"
|
|
os.environ["CUDA_VISIBLE_DEVICES"] = "0"
|
|
import glob
|
|
import random
|
|
from tqdm import tqdm
|
|
import numpy as np
|
|
import pandas as pd
|
|
from bert4keras.backend import keras, K
|
|
from bert4keras.layers import Loss
|
|
from bert4keras.models import build_transformer_model
|
|
from bert4keras.tokenizers import Tokenizer, load_vocab
|
|
from bert4keras.optimizers import Adam
|
|
from bert4keras.snippets import sequence_padding, open
|
|
from bert4keras.snippets import DataGenerator, AutoRegressiveDecoder
|
|
from keras.models import Model
|
|
import tensorflow as tf
|
|
|
|
|
|
|
|
from keras.backend import set_session
|
|
config = tf.ConfigProto()
|
|
config.gpu_options.allow_growth = True
|
|
set_session(tf.Session(config=config)) # 此处不同
|
|
global graph
|
|
graph = tf.get_default_graph()
|
|
sess = tf.Session(graph=graph)
|
|
set_session(sess)
|
|
|
|
# global graph,model
|
|
# graph = tf.get_default_graph()
|
|
# sess = tf.Session(graph=graph)
|
|
# K.set_session(sess)
|
|
|
|
|
|
# 基本参数
|
|
|
|
class Beamdataone(object):
|
|
def __init__(self, num_beams, batch_id, text, end_id, minlen, min_ends, tokenizer, output_ids):
|
|
"""
|
|
Initialize n-best list of hypotheses.
|
|
"""
|
|
self.num_beams = num_beams
|
|
self.batch_id = batch_id
|
|
self.beams = []
|
|
self.minlen = minlen
|
|
self.min_ends = min_ends
|
|
self.end_id = end_id
|
|
self.text = text
|
|
self.output_scores = np.zeros(1)
|
|
self.output_ids = output_ids
|
|
self.return_str = ""
|
|
self.over = False
|
|
self.tokenizer = tokenizer
|
|
# self.data()
|
|
self.output_str = ""
|
|
|
|
|
|
def add_data(self, step, output_probas):
|
|
'''
|
|
还存有的数据，直接可以被迭代，
|
|
@param text:
|
|
@return:
|
|
'''
|
|
# inputs = [np.array([i]) for i in inputs]
|
|
# output_ids, output_scores = self.first_output_ids, np.zeros(1)
|
|
#
|
|
# scores, states = self.predict(
|
|
# inputs, output_ids, states, temperature, 'logits'
|
|
# ) # 计算当前得分
|
|
# if step == 0: # 第1步预测后将输入重复topk次
|
|
# inputs = [np.repeat(i, self.num_beams, axis=0) for i in self.inputs]
|
|
# inputs = [self.token_ids, self.segment_ids]
|
|
# inputs = [np.array([i]) for i in inputs]
|
|
scores = output_probas
|
|
scores = self.output_scores.reshape((-1, 1)) + scores # 综合累积得分
|
|
indices = scores.argpartition(-self.num_beams, axis=None)[-self.num_beams:] # 仅保留topk
|
|
indices_1 = indices // scores.shape[1] # 行索引
|
|
indices_2 = (indices % scores.shape[1]).reshape((-1, 1)) # 列索引
|
|
self.output_ids = np.concatenate([self.output_ids[indices_1], indices_2],
|
|
1) # 更新输出
|
|
self.output_scores = np.take_along_axis(
|
|
scores, indices, axis=None
|
|
) # 更新得分
|
|
|
|
is_end = self.output_ids[:, -1] == self.end_id # 标记是否以end标记结束
|
|
self.end_counts = (self.output_ids == self.end_id).sum(1) # 统计出现的end标记
|
|
if self.output_ids.shape[1] >= self.minlen: # 最短长度判断
|
|
best = self.output_scores.argmax() # 得分最大的那个
|
|
if is_end[best] and self.end_counts[best] >= self.min_ends: # 如果已经终止
|
|
# return output_ids[best] # 直接输出
|
|
self.return_str_main(self.output_ids, best)
|
|
self.over = True
|
|
else: # 否则，只保留未完成部分
|
|
flag = ~is_end | (self.end_counts < self.min_ends) # 标记未完成序列
|
|
if not flag.all(): # 如果有已完成的
|
|
self.output_ids = self.output_ids[flag] # 扔掉已完成序列
|
|
self.output_scores = self.output_scores[flag] # 扔掉已完成序列
|
|
self.end_counts = self.end_counts[flag] # 扔掉已完成end计数
|
|
self.num_beams = flag.sum() # topk相应变化
|
|
self.output_str = [tokenizer.decode(ids) for ids in self.output_ids]
|
|
self.text = [self.text[0] for i in range(len(self.output_ids))]
|
|
|
|
|
|
# # 达到长度直接输出
|
|
# return output_ids[output_scores.argmax()]
|
|
|
|
|
|
# def data(self):
|
|
# token_ids, segment_ids = self.tokenizer.encode(self.text, maxlen=256)
|
|
# self.token_ids = token_ids
|
|
# self.segment_ids = segment_ids
|
|
|
|
|
|
# input_str = [text for i in range(self.num_beams)]
|
|
# output_str = self.output_str
|
|
# return input_str, output_str
|
|
|
|
def return_str_main(self, output_ids, best):
|
|
output_ids_best = output_ids[best]
|
|
self.return_str = self.tokenizer.decode(output_ids_best)
|
|
|
|
class TotalLoss(Loss):
|
|
"""loss分两部分，一是seq2seq的交叉熵，二是相似度的交叉熵。
|
|
"""
|
|
def compute_loss(self, inputs, mask=None):
|
|
loss1 = self.compute_loss_of_seq2seq(inputs, mask)
|
|
loss2 = self.compute_loss_of_similarity(inputs, mask)
|
|
self.add_metric(loss1, name='seq2seq_loss')
|
|
self.add_metric(loss2, name='similarity_loss')
|
|
return loss1 + loss2
|
|
|
|
def compute_loss_of_seq2seq(self, inputs, mask=None):
|
|
y_true, y_mask, _, y_pred = inputs
|
|
y_true = y_true[:, 1:] # 目标token_ids
|
|
y_mask = y_mask[:, 1:] # segment_ids，刚好指示了要预测的部分
|
|
y_pred = y_pred[:, :-1] # 预测序列，错开一位
|
|
loss = K.sparse_categorical_crossentropy(y_true, y_pred)
|
|
loss = K.sum(loss * y_mask) / K.sum(y_mask)
|
|
return loss
|
|
|
|
def compute_loss_of_similarity(self, inputs, mask=None):
|
|
_, _, y_pred, _ = inputs
|
|
y_true = self.get_labels_of_similarity(y_pred) # 构建标签
|
|
y_pred = K.l2_normalize(y_pred, axis=1) # 句向量归一化
|
|
similarities = K.dot(y_pred, K.transpose(y_pred)) # 相似度矩阵
|
|
similarities = similarities - K.eye(K.shape(y_pred)[0]) * 1e12 # 排除对角线
|
|
similarities = similarities * 30 # scale
|
|
loss = K.categorical_crossentropy(
|
|
y_true, similarities, from_logits=True
|
|
)
|
|
return loss
|
|
|
|
def get_labels_of_similarity(self, y_pred):
|
|
idxs = K.arange(0, K.shape(y_pred)[0])
|
|
idxs_1 = idxs[None, :]
|
|
idxs_2 = (idxs + 1 - idxs % 2 * 2)[:, None]
|
|
labels = K.equal(idxs_1, idxs_2)
|
|
labels = K.cast(labels, K.floatx())
|
|
return labels
|
|
|
|
|
|
class GenerateModel(object):
|
|
def __init__(self):
|
|
|
|
self.epoch_acc_vel = 0
|
|
self.config_path = r'./chinese_roberta_wwm_ext_L-12_H-768_A-12/bert_config.json'
|
|
self.checkpoint_path = r'./chinese_roberta_wwm_ext_L-12_H-768_A-12/bert_model.ckpt'
|
|
self.dict_path = r'./chinese_roberta_wwm_ext_L-12_H-768_A-12/vocab.txt'
|
|
self.maxlen = 120
|
|
|
|
def device_setup(self):
|
|
token_dict, keep_tokens = load_vocab(
|
|
dict_path=self.dict_path,
|
|
simplified=True,
|
|
startswith=['[PAD]', '[UNK]', '[CLS]', '[SEP]'],
|
|
)
|
|
tokenizer = Tokenizer(token_dict, do_lower_case=True)
|
|
|
|
# model = build_transformer_model(
|
|
# self.config_path,
|
|
# self.checkpoint_path,
|
|
# application='unilm',
|
|
# keep_tokens=keep_tokens, # 只保留keep_tokens中的字，精简原字表
|
|
# )
|
|
|
|
bert = build_transformer_model(
|
|
self.config_path,
|
|
self.checkpoint_path,
|
|
with_pool='linear',
|
|
application='unilm',
|
|
keep_tokens=keep_tokens,
|
|
return_keras_model=False,
|
|
)
|
|
|
|
encoder = keras.models.Model(bert.model.inputs, bert.model.outputs[0])
|
|
seq2seq = keras.models.Model(bert.model.inputs, bert.model.outputs[1])
|
|
|
|
# output = CrossEntropy(2)(model.inputs + model.outputs)
|
|
#
|
|
# model = Model(model.inputs, output)
|
|
# model = Model(model.inputs, model.outputs)
|
|
|
|
outputs = TotalLoss([2, 3])(bert.model.inputs + bert.model.outputs)
|
|
model = keras.models.Model(bert.model.inputs, outputs)
|
|
|
|
path_model = './output_simbert_yy/best_simbertmodel_datasim.weights'
|
|
model.load_weights(path_model)
|
|
|
|
return encoder,seq2seq, tokenizer
|
|
|
|
|
|
class CrossEntropy(Loss):
|
|
"""交叉熵作为loss，并mask掉输入部分
|
|
"""
|
|
def compute_loss(self, inputs, mask=None):
|
|
y_true, y_mask, y_pred = inputs
|
|
y_true = y_true[:, 1:] # 目标token_ids
|
|
y_mask = y_mask[:, 1:] # segment_ids，刚好指示了要预测的部分
|
|
y_pred = y_pred[:, :-1] # 预测序列，错开一位
|
|
loss = K.sparse_categorical_crossentropy(y_true, y_pred)
|
|
loss = K.sum(loss * y_mask) / K.sum(y_mask)
|
|
return loss
|
|
|
|
class AutoTitle(AutoRegressiveDecoder):
|
|
"""seq2seq解码器
|
|
"""
|
|
def __init__(self, model, tokenizer, start_id, end_id, maxlen, minlen=1):
|
|
super(AutoTitle, self).__init__(start_id, end_id, maxlen, minlen)
|
|
self.model = model
|
|
self.tokenizer = tokenizer
|
|
self.start_id = start_id
|
|
self.end_id = end_id
|
|
self.minlen = minlen
|
|
self.models = {}
|
|
if start_id is None:
|
|
self.first_output_ids = np.empty((1, 0), dtype=int)
|
|
else:
|
|
self.first_output_ids = np.array([[self.start_id]])
|
|
|
|
def data_generator(self, inputs, output_ids):
|
|
batch_token_ids, batch_segment_ids = [], []
|
|
if output_ids == []:
|
|
for txt in inputs:
|
|
token_ids, segment_ids = self.tokenizer.encode(txt, maxlen=120)
|
|
batch_token_ids.append(token_ids)
|
|
batch_segment_ids.append(segment_ids)
|
|
else:
|
|
for txt,output_id in zip(inputs, output_ids):
|
|
token_ids, segment_ids = self.tokenizer.encode(txt, output_id)
|
|
batch_token_ids.append(token_ids[:-1])
|
|
batch_segment_ids.append(segment_ids[:-1])
|
|
|
|
batch_token_ids = sequence_padding(batch_token_ids)
|
|
batch_segment_ids = sequence_padding(batch_segment_ids)
|
|
return batch_token_ids, batch_segment_ids
|
|
|
|
|
|
def beam_search_batch(
|
|
self,
|
|
inputs_str,
|
|
states=None,
|
|
temperature=1,
|
|
min_ends=1,
|
|
num_beam=3
|
|
):
|
|
"""随机采样n个结果
|
|
说明：非None的topk表示每一步只从概率最高的topk个中采样；而非None的topp
|
|
表示每一步只从概率最高的且概率之和刚好达到topp的若干个token中采样。
|
|
返回：n个解码序列组成的list。
|
|
"""
|
|
output_str = []
|
|
# token_ids, segment_ids = self.data_generator(inputs, output_ids)
|
|
batch_nums = len(inputs_str)
|
|
return_str_batch = [0] * batch_nums
|
|
# output_ids = np.empty((batch_nums, 0), dtype=int)
|
|
output_ids = np.empty((1, 0), dtype=int)
|
|
generated = [Beamdataone(num_beam, i, [inputs_str[i]], self.end_id, self.minlen, min_ends, self.tokenizer, output_ids) for i in range(batch_nums)]
|
|
# index_data = [i for i in range(batch_nums)]
|
|
for step in range(self.maxlen):
|
|
# if step == 0:
|
|
# token_ids, segment_ids = self.data_generator(inputs_str, output_str)
|
|
# else:
|
|
# inputs_str, output_str = [], []
|
|
text_batch, output_str_batch = [], []
|
|
batch_input_num_beam_num = []
|
|
for i in generated:
|
|
text = i.text
|
|
text_batch.extend(text)
|
|
if i.output_str != "":
|
|
output_str_batch.extend(i.output_str)
|
|
if step != 0:
|
|
batch_input_num_beam_num.append(i.num_beams)
|
|
|
|
token_ids, segment_ids = self.data_generator(text_batch, output_str_batch)
|
|
|
|
# token_ids_batch = sequence_padding(token_ids_batch)
|
|
# segment_ids_batch = sequence_padding(segment_ids_batch)
|
|
# output_ids_batch = np.array(output_ids_batch)
|
|
# if step == 0:
|
|
|
|
inputs = [token_ids, segment_ids]
|
|
|
|
probas = self.predict_batch(
|
|
inputs
|
|
) # 计算当前概率
|
|
|
|
probas_new = []
|
|
probas_bool = np.array(token_ids, dtype=bool)
|
|
# np.array(np.where(probas_bool == True))
|
|
for i, sentence in enumerate(probas_bool):
|
|
lie = np.array(np.where(sentence == True))[0]
|
|
probas_new.append(probas[i, lie[-1]])
|
|
probas = np.array(probas_new)
|
|
|
|
|
|
if step != 0:
|
|
num = 0
|
|
if len(generated) > 1:
|
|
index = 0
|
|
for index in range(len(batch_input_num_beam_num)-1):
|
|
cc = num
|
|
num += batch_input_num_beam_num[index]
|
|
generated[index].add_data(step, probas[cc:num,:])
|
|
generated[index+1].add_data(step, probas[num:,:])
|
|
else:
|
|
generated[0].add_data(step, probas[:,:])
|
|
|
|
else:
|
|
for index in range(len(generated)):
|
|
generated[index].add_data(step, probas[index,:])
|
|
# i = 0
|
|
# while True:
|
|
# bool_ = generated[i].over
|
|
# if bool_ == True:
|
|
# one_sentence = generated.pop(i)
|
|
# return_str_batch[i] = one_sentence.return_str
|
|
# if i > len(generated) - 1:
|
|
# break
|
|
# else:
|
|
# i += 1
|
|
# if i > len(generated) - 1:
|
|
# break
|
|
|
|
generated_new = []
|
|
for i in range(len(generated)):
|
|
bool_ = generated[i].over
|
|
if bool_ == False:
|
|
generated_new.append(generated[i])
|
|
else:
|
|
return_str_batch[generated[i].batch_id] = generated[i].return_str
|
|
generated = generated_new
|
|
|
|
|
|
if generated == []:
|
|
return return_str_batch
|
|
return return_str_batch
|
|
|
|
|
|
def random_sample_batch(
|
|
self,
|
|
inputs,
|
|
n,
|
|
topk=None,
|
|
topp=None,
|
|
states=None,
|
|
temperature=1,
|
|
min_ends=1
|
|
):
|
|
"""随机采样n个结果
|
|
说明：非None的topk表示每一步只从概率最高的topk个中采样；而非None的topp
|
|
表示每一步只从概率最高的且概率之和刚好达到topp的若干个token中采样。
|
|
返回：n个解码序列组成的list。
|
|
"""
|
|
inputs = [np.array([i for j in i]) for i in inputs]
|
|
output_ids = self.first_output_ids
|
|
results = []
|
|
for step in range(self.maxlen):
|
|
probas, states = self.predict(
|
|
inputs, output_ids, states, temperature, 'probas'
|
|
) # 计算当前概率
|
|
probas /= probas.sum(axis=1, keepdims=True) # 确保归一化
|
|
if step == 0: # 第1步预测后将结果重复n次
|
|
probas = np.repeat(probas, n, axis=0)
|
|
inputs = [np.repeat(i, n, axis=0) for i in inputs]
|
|
output_ids = np.repeat(output_ids, n, axis=0)
|
|
if topk is not None:
|
|
k_indices = probas.argpartition(-topk,
|
|
axis=1)[:, -topk:] # 仅保留topk
|
|
probas = np.take_along_axis(probas, k_indices, axis=1) # topk概率
|
|
probas /= probas.sum(axis=1, keepdims=True) # 重新归一化
|
|
if topp is not None:
|
|
p_indices = probas.argsort(axis=1)[:, ::-1] # 从高到低排序
|
|
probas = np.take_along_axis(probas, p_indices, axis=1) # 排序概率
|
|
cumsum_probas = np.cumsum(probas, axis=1) # 累积概率
|
|
flag = np.roll(cumsum_probas >= topp, 1, axis=1) # 标记超过topp的部分
|
|
flag[:, 0] = False # 结合上面的np.roll，实现平移一位的效果
|
|
probas[flag] = 0 # 后面的全部置零
|
|
probas /= probas.sum(axis=1, keepdims=True) # 重新归一化
|
|
sample_func = lambda p: np.random.choice(len(p), p=p) # 按概率采样函数
|
|
sample_ids = np.apply_along_axis(sample_func, 1, probas) # 执行采样
|
|
sample_ids = sample_ids.reshape((-1, 1)) # 对齐形状
|
|
if topp is not None:
|
|
sample_ids = np.take_along_axis(
|
|
p_indices, sample_ids, axis=1
|
|
) # 对齐原id
|
|
if topk is not None:
|
|
sample_ids = np.take_along_axis(
|
|
k_indices, sample_ids, axis=1
|
|
) # 对齐原id
|
|
output_ids = np.concatenate([output_ids, sample_ids], 1) # 更新输出
|
|
is_end = output_ids[:, -1] == self.end_id # 标记是否以end标记结束
|
|
end_counts = (output_ids == self.end_id).sum(1) # 统计出现的end标记
|
|
if output_ids.shape[1] >= self.minlen: # 最短长度判断
|
|
flag = is_end & (end_counts >= min_ends) # 标记已完成序列
|
|
if flag.any(): # 如果有已完成的
|
|
for ids in output_ids[flag]: # 存好已完成序列
|
|
results.append(ids)
|
|
flag = (flag == False) # 标记未完成序列
|
|
inputs = [i[flag] for i in inputs] # 只保留未完成部分输入
|
|
output_ids = output_ids[flag] # 只保留未完成部分候选集
|
|
end_counts = end_counts[flag] # 只保留未完成部分end计数
|
|
if len(output_ids) == 0:
|
|
break
|
|
# 如果还有未完成序列，直接放入结果
|
|
for ids in output_ids:
|
|
results.append(ids)
|
|
# 返回结果
|
|
return results
|
|
|
|
def random_sample_and_beam_search(
|
|
self,
|
|
inputs,
|
|
n,
|
|
topk=None,
|
|
topp=None,
|
|
states=None,
|
|
temperature=1,
|
|
min_ends=1
|
|
):
|
|
"""随机采样n个结果
|
|
说明：非None的topk表示每一步只从概率最高的topk个中采样；而非None的topp
|
|
表示每一步只从概率最高的且概率之和刚好达到topp的若干个token中采样。
|
|
返回：n个解码序列组成的list。
|
|
"""
|
|
whether_end_b = False
|
|
results_r = []
|
|
results_b = []
|
|
# index_r = [i for i in range(n)]
|
|
# index_b = [i for i in range(topk)]
|
|
index_r = np.arange(n)
|
|
index_b = np.arange(topk)
|
|
inputs = [np.array([i]) for i in inputs]
|
|
output_ids, output_scores = self.first_output_ids, np.zeros(1)
|
|
results = []
|
|
for step in range(self.maxlen):
|
|
beam_n = len(index_b)
|
|
probas, states = self.predict(
|
|
inputs, output_ids, states, temperature, 'probas'
|
|
) # 计算当前概率
|
|
probas = probas / probas.sum(axis=1, keepdims=True) # 确保归一化
|
|
if step == 0: # 第1步预测后将结果重复n次
|
|
probas = np.repeat(probas, n + topk, axis=0)
|
|
inputs_r = [np.repeat(i, n, axis=0) for i in inputs]
|
|
output_ids = np.repeat(output_ids, n + topk, axis=0)
|
|
inputs_b = [np.repeat(i, topk, axis=0) for i in inputs]
|
|
else:
|
|
if whether_end_b == False:
|
|
inputs_r = [i[:-beam_n, :] for i in inputs]
|
|
inputs_b = [i[-beam_n:, :] for i in inputs]
|
|
else:
|
|
inputs_r = inputs
|
|
if whether_end_b == False:
|
|
probas_r = probas[:-beam_n, :]
|
|
else:
|
|
probas_r = probas
|
|
if step == 0:
|
|
probas_b = probas[0,:]
|
|
else:
|
|
probas_b = probas[-beam_n:, :]
|
|
|
|
if whether_end_b == False:
|
|
output_ids_r = output_ids[:-beam_n, :]
|
|
output_ids_b = output_ids[-beam_n:, :]
|
|
else:
|
|
output_ids_r = output_ids
|
|
k_indices = probas_r.argpartition(-topk,
|
|
axis=1)[:, -topk:] # 仅保留topk
|
|
probas_r = np.take_along_axis(probas_r, k_indices, axis=1) # topk概率
|
|
probas_r /= probas_r.sum(axis=1, keepdims=True) # 重新归一化
|
|
|
|
if whether_end_b == False:
|
|
scores = output_scores.reshape((-1, 1)) + probas_b # 综合累积得分
|
|
indices = scores.argpartition(-topk, axis=None)[-topk:] # 仅保留topk
|
|
indices_1 = indices // scores.shape[1] # 行索引
|
|
indices_2 = (indices % scores.shape[1]).reshape((-1, 1)) # 列索引
|
|
try:
|
|
output_ids_b = np.concatenate([output_ids_b[indices_1], indices_2],
|
|
1) # 更新输出
|
|
except:
|
|
print(output_ids_b.shape)
|
|
print(indices_1)
|
|
print(indices_2)
|
|
exit()
|
|
output_scores = np.take_along_axis(
|
|
scores, indices, axis=None
|
|
) # 更新得分
|
|
sample_func = lambda p: np.random.choice(len(p), p=p) # 按概率采样函数
|
|
try:
|
|
sample_ids = np.apply_along_axis(sample_func, 1, probas_r) # 执行采样
|
|
except:
|
|
print(probas_r)
|
|
sample_ids = sample_ids.reshape((-1, 1)) # 对齐形状
|
|
if topk is not None:
|
|
sample_ids = np.take_along_axis(
|
|
k_indices, sample_ids, axis=1
|
|
) # 对齐原id
|
|
output_ids_r = np.concatenate([output_ids_r, sample_ids], 1) # 更新输出
|
|
|
|
# output_ids = np.concatenate([output_ids_r, output_ids_b], 0)
|
|
if whether_end_b == False:
|
|
is_end_r = output_ids_r[:, -1] == self.end_id # 标记是否以end标记结束
|
|
is_end_b = output_ids_b[:, -1] == self.end_id # 标记是否以end标记结束
|
|
else:
|
|
is_end_r = output_ids_r[:, -1] == self.end_id
|
|
|
|
if whether_end_b == False:
|
|
end_counts_r = (output_ids_r == self.end_id).sum(1) # 统计出现的end标记
|
|
end_counts_b = (output_ids_b == self.end_id).sum(1) # 统计出现的end标记
|
|
else:
|
|
end_counts_r = (output_ids_r == self.end_id).sum(1)
|
|
# random_serach
|
|
if output_ids_r.shape[1] >= self.minlen: # 最短长度判断
|
|
flag = is_end_r & (end_counts_r >= min_ends) # 标记已完成序列
|
|
if flag.any(): # 如果有已完成的
|
|
for ids in output_ids_r[flag]: # 存好已完成序列
|
|
results_r.append(ids)
|
|
flag = (flag == False) # 标记未完成序列
|
|
try:
|
|
index_r = index_r[flag]
|
|
except:
|
|
print("flag",flag)
|
|
print("index_r",index_r)
|
|
inputs_r = [i[flag] for i in inputs_r] # 只保留未完成部分输入
|
|
output_ids_r = output_ids_r[flag] # 只保留未完成部分候选集
|
|
end_counts_r = end_counts_r[flag] # 只保留未完成部分end计数
|
|
|
|
# beam_serach
|
|
if whether_end_b == False:
|
|
if output_ids_b.shape[1] >= self.minlen: # 最短长度判断
|
|
best = output_scores.argmax() # 得分最大的那个
|
|
if is_end_b[best] and end_counts_b[best] >= min_ends: # 如果已经终止
|
|
results_b.append(output_ids_b[best]) # 直接输出
|
|
whether_end_b = True
|
|
else: # 否则，只保留未完成部分
|
|
flag_b = ~is_end_b | (end_counts_b < min_ends) # 标记未完成序列
|
|
if not flag_b.all(): # 如果有已完成的
|
|
index_b = index_b[flag_b]
|
|
inputs_b = [i[flag_b] for i in inputs_b] # 扔掉已完成序列
|
|
output_ids_b = output_ids_b[flag_b] # 扔掉已完成序列
|
|
output_scores = output_scores[flag_b] # 扔掉已完成序列
|
|
end_counts_b = end_counts_b[flag_b] # 扔掉已完成end计数
|
|
topk = flag_b.sum() # topk相应变化
|
|
|
|
if whether_end_b == False and len(output_ids_r) != 0:
|
|
token_r = inputs_r[0]
|
|
sample_ids_r = inputs_r[1]
|
|
token_b = inputs_b[0]
|
|
sample_ids_b = inputs_b[1]
|
|
token = np.concatenate([token_r,token_b],0)
|
|
sample_ids = np.concatenate([sample_ids_r,sample_ids_b],0)
|
|
inputs = [token,sample_ids]
|
|
output_ids = np.concatenate([output_ids_r, output_ids_b], 0)
|
|
elif whether_end_b == True and len(output_ids_r) != 0:
|
|
inputs = inputs_r
|
|
output_ids = output_ids_r
|
|
elif whether_end_b == False and len(output_ids_r) == 0:
|
|
inputs = inputs_b
|
|
output_ids = output_ids_b
|
|
else:
|
|
break
|
|
|
|
|
|
# 如果还有未完成序列，直接放入结果
|
|
for ids in output_ids:
|
|
results.append(ids)
|
|
# 返回结果
|
|
return results_r, results_b
|
|
|
|
|
|
def top_batch(
|
|
self,
|
|
inputs_str,
|
|
temperature=1,
|
|
min_ends=1
|
|
):
|
|
"""随机采样n个结果
|
|
说明：非None的topk表示每一步只从概率最高的topk个中采样；而非None的topp
|
|
表示每一步只从概率最高的且概率之和刚好达到topp的若干个token中采样。
|
|
返回：n个解码序列组成的list。
|
|
"""
|
|
output_str = []
|
|
# token_ids, segment_ids = self.data_generator(inputs, output_ids)
|
|
batch_nums = len(inputs_str)
|
|
output_ids =self.first_output_ids_batch = np.empty((batch_nums, 0), dtype=int)
|
|
|
|
results = [[] for i in range(batch_nums)]
|
|
index_data = [i for i in range(batch_nums)]
|
|
for step in range(self.maxlen):
|
|
|
|
token_ids, segment_ids = self.data_generator(inputs_str, output_str)
|
|
inputs = [token_ids, segment_ids]
|
|
probas = self.predict_batch(
|
|
inputs
|
|
) # 计算当前概率
|
|
# probas /= probas.sum(axis=1, keepdims=True) # 确保归一化
|
|
|
|
probas_new = []
|
|
probas_bool = np.array(token_ids, dtype=bool)
|
|
# np.array(np.where(probas_bool == True))
|
|
for i,sentence in enumerate(probas_bool):
|
|
lie = np.array(np.where(sentence == True))[0]
|
|
probas_new.append(probas[i,lie[-1]])
|
|
probas = np.array(probas_new)
|
|
k_indices = np.argmax(probas,axis=1) # 仅保留topk
|
|
k_indices = k_indices.reshape(-1,1)
|
|
|
|
sample_ids = k_indices
|
|
output_ids = np.concatenate([output_ids, sample_ids], 1) # 更新输出
|
|
is_end = output_ids[:, -1] == self.end_id # 标记是否以end标记结束
|
|
end_counts = (output_ids == self.end_id).sum(1) # 统计出现的end标记
|
|
if output_ids.shape[1] >= self.minlen: # 最短长度判断
|
|
flag = is_end & (end_counts >= min_ends) # 标记已完成序列
|
|
if flag.any(): # 如果有已完成的
|
|
index = np.array(np.where(flag == True))[0]
|
|
pop_index = []
|
|
for i in index:
|
|
results[index_data[i]] = output_ids[i]
|
|
pop_index.append(index_data[i])
|
|
for i in pop_index:
|
|
index_data.remove(i)
|
|
# for ids in output_ids[flag]: # 存好已完成序列
|
|
# results.append(ids)
|
|
flag = (flag == False) # 标记未完成序列
|
|
inputs_str = [inputs_str[i] for i in index_data] # 只保留未完成部分输入
|
|
output_ids = output_ids[flag] # 只保留未完成部分候选集
|
|
if len(output_ids) == 0:
|
|
break
|
|
else:
|
|
output_str = [tokenizer.decode(ids) for ids in output_ids]
|
|
else:
|
|
output_str = [tokenizer.decode(ids) for ids in output_ids]
|
|
# 如果还有未完成序列，直接放入结果
|
|
# for ids in output_ids:
|
|
# results.append(ids)
|
|
# 返回结果
|
|
return results
|
|
|
|
|
|
@AutoRegressiveDecoder.wraps(default_rtype='probas')
|
|
def predict(self, inputs, output_ids, states):
|
|
token_ids, segment_ids = inputs
|
|
token_ids = np.concatenate([token_ids, output_ids], 1)
|
|
segment_ids = np.concatenate([segment_ids, np.ones_like(output_ids)], 1)
|
|
with graph.as_default():
|
|
K.set_session(sess)
|
|
nodes = self.last_token(self.model).predict([token_ids, segment_ids])
|
|
return nodes
|
|
# return self.last_token(self.model).predict([token_ids, segment_ids])
|
|
|
|
def predict_batch(self, inputs):
|
|
token_ids, segment_ids = inputs
|
|
# token_ids = np.concatenate([token_ids, output_ids], 1)
|
|
# segment_ids = np.concatenate([segment_ids, np.ones_like(output_ids)], 1)
|
|
with graph.as_default():
|
|
K.set_session(sess)
|
|
nodes = self.model.predict([token_ids, segment_ids])
|
|
return nodes
|
|
# return self.last_token(self.model).predict([token_ids, segment_ids])
|
|
|
|
def generate(self, text, topk=5):
|
|
token_ids, segment_ids = self.tokenizer.encode(text, maxlen=256)
|
|
output_ids = self.beam_search([token_ids, segment_ids],
|
|
topk=topk) # 基于beam search
|
|
return self.tokenizer.decode(output_ids)
|
|
|
|
def generate_random(self, text, n=20, topk=5):
|
|
if isinstance(text, list):
|
|
text = text[0]
|
|
token_ids, segment_ids = self.tokenizer.encode(text, maxlen=120)
|
|
output_ids = self.random_sample([token_ids, segment_ids], n, topk) # 基于随机采样
|
|
return [tokenizer.decode(ids) for ids in output_ids]
|
|
|
|
def generate_random_topp(self, text, n=20, topp=0.98):
|
|
if isinstance(text, list):
|
|
text = text[0]
|
|
token_ids, segment_ids = self.tokenizer.encode(text, maxlen=120)
|
|
output_ids = self.random_sample([token_ids, segment_ids], n, topp=topp) # 基于随机采样
|
|
return [tokenizer.decode(ids) for ids in output_ids]
|
|
|
|
def generate_top(self, text):
|
|
output_ids = self.top_batch(text) # 基于随机采样
|
|
return [tokenizer.decode(ids) for ids in output_ids]
|
|
|
|
def generate_beam_search_batch(self, text):
|
|
output_str = self.beam_search_batch(text) # 基于随机采样
|
|
return output_str
|
|
|
|
def generate_random_sample_and_beam_search(self, text, n=20, topk=5):
|
|
text = text[0]
|
|
token_ids, segment_ids = self.tokenizer.encode(text, maxlen=120)
|
|
output_ids_r, output_ids_b = self.random_sample_and_beam_search([token_ids, segment_ids], n=n,
|
|
topk=topk) # 基于beam search
|
|
output_str_r = [self.tokenizer.decode(ids) for ids in output_ids_r]
|
|
output_str_b = [self.tokenizer.decode(ids) for ids in output_ids_b]
|
|
return output_str_r, output_str_b
|
|
|
|
def gen_synonyms(self, text, n=20):
|
|
""""含义： 产生sent的n个相似句，然后返回最相似的k个。
|
|
做法：用seq2seq生成，并用encoder算相似度并排序。
|
|
"""
|
|
r = self.generate_random_topp(text, n)
|
|
r = [i for i in set(r) if i != text]
|
|
r = [text] + r
|
|
X, S = [], []
|
|
for t in r:
|
|
x, s = tokenizer.encode(t)
|
|
X.append(x)
|
|
S.append(s)
|
|
X = sequence_padding(X)
|
|
S = sequence_padding(S)
|
|
Z = encoder.predict([X, S])
|
|
Z /= (Z ** 2).sum(axis=1, keepdims=True) ** 0.5
|
|
argsort = np.dot(Z[1:], -Z[0]).argsort()
|
|
return [r[i + 1] for i in argsort]
|
|
|
|
|
|
def gen_synonyms_short(self, text, n=20, len_s = 0.9):
|
|
""""含义： 产生sent的n个相似句，然后返回最相似的k个。
|
|
做法：用seq2seq生成，并用encoder算相似度并排序。
|
|
"""
|
|
if isinstance(text, list):
|
|
text = text[0]
|
|
new_text_len = int(len(text) * len_s)
|
|
r = self.generate_random(text, n)
|
|
r = [i for i in set(r) if i != text]
|
|
r = [text] + r
|
|
X, S = [], []
|
|
for t in r:
|
|
x, s = tokenizer.encode(t)
|
|
X.append(x)
|
|
S.append(s)
|
|
X = sequence_padding(X)
|
|
S = sequence_padding(S)
|
|
with graph.as_default():
|
|
K.set_session(sess)
|
|
Z = encoder.predict([X, S])
|
|
Z /= (Z ** 2).sum(axis=1, keepdims=True) ** 0.5
|
|
argsort = np.dot(Z[1:], -Z[0]).argsort()
|
|
sentence_list = [r[i + 1] for i in argsort]
|
|
|
|
return_list = []
|
|
for i in sentence_list:
|
|
if len(i) < new_text_len:
|
|
return_list.append(i)
|
|
break
|
|
|
|
for i in sentence_list:
|
|
if new_text_len <len(i) < len(text):
|
|
return_list.append(i)
|
|
break
|
|
if return_list != []:
|
|
return return_list[0]
|
|
else:
|
|
return sentence_list[0]
|
|
|
|
generatemodel = GenerateModel()
|
|
encoder,seq2seq, tokenizer = generatemodel.device_setup()
|
|
autotitle = AutoTitle(seq2seq, tokenizer, start_id=None, end_id=tokenizer._token_end_id, maxlen=120)
|
|
|
|
|
|
|
|
|
|
def just_show(file):
|
|
data = []
|
|
try:
|
|
with open(file, 'r', encoding="utf-8") as f:
|
|
lines = [x.strip() for x in f if x.strip() != '']
|
|
except:
|
|
with open(file, 'r', encoding="gbk") as f:
|
|
lines = [x.strip() for x in f if x.strip() != '']
|
|
# s2 = u'她只能应下来。'
|
|
# lines = pd.read_csv(file,encoding="gbk").values.tolist()
|
|
# random.shuffle(lines)
|
|
# lines = lines[:20]
|
|
for s in tqdm(lines[:2]):
|
|
print(s)
|
|
pre = autotitle.generate_random(s)
|
|
print(s)
|
|
print(pre)
|
|
# data.append([s, pre])
|
|
# pd.DataFrame(data,columns=["原始文本","生成文本"]).to_csv("data/text_测试一万字_unilm_修正数据_小说预训练_全部数据_epoch72_反向训练.csv")
|
|
|
|
|
|
def just_show_sentence(file: list) -> object:
|
|
"""
|
|
@param file:list
|
|
"""
|
|
text = file[0]
|
|
pre = autotitle.generate(text)
|
|
return pre
|
|
|
|
def just_show_sentence_batch_top(file: list) -> object:
|
|
text = file
|
|
pre = autotitle.generate_top(text)
|
|
return pre
|
|
|
|
def just_show_csv_random(file):
|
|
data_new = []
|
|
data = pd.read_csv(file).values.tolist()
|
|
for sentence in tqdm(data):
|
|
sentence = sentence[1]
|
|
print(sentence)
|
|
data_new_dan = []
|
|
data_new_dan.extend([sentence, len(sentence)])
|
|
pre = autotitle.generate_random(sentence)
|
|
for i in pre:
|
|
data_new_dan.extend([i, len(i)])
|
|
|
|
data_new.append(data_new_dan)
|
|
pd.DataFrame(data_new).to_csv("data/###第3章 非常尴尬_sim_topK_5.csv")
|
|
# return pre
|
|
|
|
|
|
def just_show_chachong_random(file):
|
|
text = file[0]
|
|
pre = autotitle.gen_synonyms(text)
|
|
return pre
|
|
|
|
def just_show_sentence_batch(file: list) -> object:
|
|
text = file
|
|
pre = autotitle.generate_beam_search_batch(text)
|
|
return pre
|
|
|
|
def just_show_csv_beam(file):
|
|
data_new = []
|
|
data = pd.read_csv(file).values.tolist()
|
|
for sentence in tqdm(data):
|
|
sentence = sentence[1]
|
|
print(sentence)
|
|
data_new_dan = []
|
|
data_new_dan.extend([sentence, len(sentence)])
|
|
pre = autotitle.generate([sentence])
|
|
print(pre)
|
|
data_new_dan.extend([pre, len(pre)])
|
|
data_new.append(data_new_dan)
|
|
pd.DataFrame(data_new).to_csv("data/###第3章 非常尴尬_sim_topK_1.csv")
|
|
|
|
|
|
def chulichangju_1(text, snetence_id, chulipangban_return_list, short_num):
|
|
fuhao = ["，","？","！","…"]
|
|
text_1 = text[:120]
|
|
text_2 = text[120:]
|
|
text_1_new = ""
|
|
for i in range(len(text_1)-1, -1, -1):
|
|
if text_1[i] in fuhao:
|
|
text_1_new = text_1[:i]
|
|
text_1_new += text_1[i]
|
|
chulipangban_return_list.append([text_1_new, snetence_id, short_num])
|
|
if text_2 != "":
|
|
if i+1 != 120:
|
|
text_2 = text_1[i+1:] + text_2
|
|
break
|
|
# else:
|
|
# chulipangban_return_list.append(text_1)
|
|
if text_1_new == "":
|
|
chulipangban_return_list.append([text_1, snetence_id, short_num])
|
|
if text_2 != "":
|
|
short_num += 1
|
|
chulipangban_return_list = chulichangju_1(text_2, snetence_id, chulipangban_return_list, short_num)
|
|
return chulipangban_return_list
|
|
|
|
|
|
def chulipangban_test_1(text, snetence_id):
|
|
sentence_list = text.split("。")
|
|
sentence_list_new = []
|
|
for i in sentence_list:
|
|
if i != "":
|
|
sentence_list_new.append(i)
|
|
sentence_list = sentence_list_new
|
|
sentence_batch_list = []
|
|
sentence_batch_one = []
|
|
sentence_batch_length = 0
|
|
return_list = []
|
|
for sentence in sentence_list:
|
|
if len(sentence) < 120:
|
|
sentence_batch_length += len(sentence)
|
|
sentence_batch_list.append([sentence, snetence_id, 0])
|
|
# sentence_pre = autotitle.gen_synonyms_short(sentence)
|
|
# return_list.append(sentence_pre)
|
|
else:
|
|
sentence_split_list = chulichangju_1(sentence, snetence_id, [], 0)
|
|
for sentence_short in sentence_split_list:
|
|
sentence_batch_list.append(sentence_short)
|
|
return sentence_batch_list
|
|
|
|
|
|
def paragraph_test_(text:list, text_new:list):
|
|
|
|
for i in range(len(text)):
|
|
text = chulipangban_test_1(text, i)
|
|
text = "。".join(text)
|
|
text_new.append(text)
|
|
|
|
# text_new_str = "".join(text_new)
|
|
return text_new
|
|
|
|
def paragraph_test(text:list):
|
|
|
|
text_new = []
|
|
for i in range(len(text)):
|
|
text_list = chulipangban_test_1(text[i], i)
|
|
text_new.extend(text_list)
|
|
|
|
# text_new_str = "".join(text_new)
|
|
return text_new
|
|
|
|
|
|
def batch_data_process(text_list):
|
|
sentence_batch_length = 0
|
|
sentence_batch_one = []
|
|
sentence_batch_list = []
|
|
|
|
for sentence in text_list:
|
|
sentence_batch_length += len(sentence[0])
|
|
sentence_batch_one.append(sentence)
|
|
if sentence_batch_length > 500:
|
|
sentence_batch_length = 0
|
|
sentence_ = sentence_batch_one.pop(-1)
|
|
sentence_batch_list.append(sentence_batch_one)
|
|
sentence_batch_one = []
|
|
sentence_batch_one.append(sentence_)
|
|
sentence_batch_list.append(sentence_batch_one)
|
|
return sentence_batch_list
|
|
|
|
def batch_predict(batch_data_list):
|
|
batch_data_list_new = []
|
|
batch_data_text_list = []
|
|
batch_data_snetence_id_list = []
|
|
for i in batch_data_list:
|
|
batch_data_text_list.append(i[0])
|
|
batch_data_snetence_id_list.append(i[1:])
|
|
batch_pre_data_list = autotitle.generate_beam_search_batch(batch_data_text_list)
|
|
# batch_pre_data_list = batch_data_text_list
|
|
for text,sentence_id in zip(batch_pre_data_list,batch_data_snetence_id_list):
|
|
batch_data_list_new.append([text] + sentence_id)
|
|
|
|
return batch_data_list_new
|
|
|
|
|
|
def predict_data_post_processing(text_list):
|
|
text_list_sentence = []
|
|
# text_list_sentence.append([text_list[0][0], text_list[0][1]])
|
|
for i in range(len(text_list)):
|
|
if text_list[i][2] != 0:
|
|
text_list_sentence[-1][0] += str(text_list[i][0])
|
|
else:
|
|
text_list_sentence.append([text_list[i][0], text_list[i][1]])
|
|
|
|
return_list = []
|
|
sentence_one = []
|
|
sentence_id = 0
|
|
for i in text_list_sentence:
|
|
if i[1] == sentence_id:
|
|
sentence_one.append(i[0])
|
|
else:
|
|
sentence_id = i[1]
|
|
return_list.append("。".join(sentence_one))
|
|
sentence_one = []
|
|
sentence_one.append(i[0])
|
|
if sentence_one != []:
|
|
return_list.append("。".join(sentence_one))
|
|
return return_list
|
|
|
|
|
|
def main(text:list):
|
|
text_list = paragraph_test(text[:10])
|
|
batch_data = batch_data_process(text_list)
|
|
text_list = []
|
|
for i in tqdm(batch_data):
|
|
pre = batch_predict(i)
|
|
text_list.extend(pre)
|
|
return_list = predict_data_post_processing(text_list)
|
|
return return_list
|
|
|
|
def chulichangju_2(text, chulipangban_return_list):
|
|
fuhao = ["，","？","！","…"]
|
|
text_1 = text[:120]
|
|
text_2 = text[120:]
|
|
text_1_new = ""
|
|
for i in range(len(text_1)-1, -1, -1):
|
|
if text_1[i] in fuhao:
|
|
text_1_new = text_1[:i]
|
|
text_1_new += text_1[i]
|
|
chulipangban_return_list.append(text_1_new)
|
|
if text_2 != "":
|
|
if i+1 != 120:
|
|
text_2 = text_1[i+1:] + text_2
|
|
break
|
|
# else:
|
|
# chulipangban_return_list.append(text_1)
|
|
if text_1_new == "":
|
|
chulipangban_return_list.append(text_1)
|
|
if text_2 != "":
|
|
chulipangban_return_list = chulichangju_2(text_2, chulipangban_return_list)
|
|
return chulipangban_return_list
|
|
|
|
if __name__ == '__main__':
|
|
|
|
text = ["强调轻资产经营, 更加重视经营风险的“规避”", "历史和当下都证明,创新是民族生存、发展的不竭源泉,是是自身发展的必然选择", "是时代对于青年们的深切呼唤"]
|
|
print(just_show_sentence_batch(text))
|
|
# print(type(just_show_sentence_batch(text)))
|
|
|
|
|
|
# path = "./data/700条论文测试.xlsx"
|
|
# df_list = pd.read_excel(path).values.tolist()
|
|
#
|
|
# df_list_new = []
|
|
# print(len(df_list))
|
|
# for i in tqdm(df_list):
|
|
# try:
|
|
# pre = just_show_sentence([i[0]])
|
|
# df_list_new.append([i[0], i[1]] + [pre])
|
|
# except:
|
|
# print(i[0])
|
|
# continue
|
|
# df = pd.DataFrame(df_list_new)
|
|
# df.to_excel("./data/700条论文测试_18.xlsx", index=None)
|
|
|
|
# path = './data/11篇csv'
|
|
# path_list = []
|
|
# for file_name in os.listdir(path):
|
|
# path_list.append(file_name)
|
|
# for docx_name in path_list:
|
|
# df_list_new = []
|
|
# df = pd.read_csv(path + "/" + docx_name).values.tolist()
|
|
# for dan in tqdm(df):
|
|
# pre = just_show_sentence([dan[0]])
|
|
# df_list_new.append([dan[0],pre])
|
|
# df = pd.DataFrame(df_list_new)
|
|
# file_name = docx_name.split(".")[0]
|
|
# df.to_excel("./data/11篇excel/{}_.xlsx".format(file_name), index=None)
|
|
|
|
#+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
|
|
# path = './data/11篇txt'
|
|
# path_new = './data/11篇model1'
|
|
# path_list = []
|
|
# df_list_new = []
|
|
# for file_name in os.listdir(path):
|
|
# path_list.append(file_name)
|
|
# for docx_name in path_list:
|
|
# with open(path + "/" + docx_name, 'r', encoding="utf-8") as f:
|
|
# lines = [x.strip() for x in f if x.strip() != '']
|
|
# # for dan in tqdm(lines):
|
|
# #
|
|
# # pre = just_show_sentence([dan])
|
|
# # df_list_new.append(pre)
|
|
# for i in tqdm(main(lines)):
|
|
# print(i)
|
|
# df_list_new.append(i)
|
|
#
|
|
#
|
|
# with open(path_new + "/" + docx_name, "w", encoding='utf-8') as file:
|
|
# for i in df_list_new:
|
|
# file.write(i + '\n')
|
|
# file.close()
|
|
|
|
# +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
|
|
# path = './data/11篇txt'
|
|
# path_new = './data/11篇model1'
|
|
# path_list = []
|
|
#
|
|
# for file_name in os.listdir(path):
|
|
# path_list.append(file_name)
|
|
# for docx_name in path_list:
|
|
# df_list_new = []
|
|
# with open(path + "/" + docx_name, 'r', encoding="utf-8") as f:
|
|
# lines = [x.strip() for x in f if x.strip() != '']
|
|
# for dan in tqdm(lines):
|
|
# break_ = False
|
|
# for i in dan:
|
|
# if i == "章":
|
|
# break_ = True
|
|
# break
|
|
# if break_ == True:
|
|
# df_list_new.append(dan)
|
|
# continue
|
|
# pre = just_show_sentence([dan])
|
|
# df_list_new.append(pre)
|
|
#
|
|
#
|
|
#
|
|
# with open(path_new + "/" + docx_name, "w", encoding='utf-8') as file:
|
|
# for i in df_list_new:
|
|
# file.write(i + '\n')
|
|
# file.close()
|
|
|
|
|