#! -*- coding: utf-8 -*-

import os
# os.environ["TF_KERAS"] = "1"
os.environ["CUDA_VISIBLE_DEVICES"] = "0"
import glob
import random
from tqdm import tqdm
import numpy as np
import pandas as pd
from bert4keras.backend import keras, K
from bert4keras.layers import Loss
from bert4keras.models import build_transformer_model
from bert4keras.tokenizers import Tokenizer, load_vocab
from bert4keras.optimizers import Adam
from bert4keras.snippets import sequence_padding, open
from bert4keras.snippets import DataGenerator, AutoRegressiveDecoder
from keras.models import Model
import tensorflow as tf



from keras.backend import set_session
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
set_session(tf.Session(config=config)) # 此处不同
global graph
graph = tf.get_default_graph()
sess = tf.Session(graph=graph)
set_session(sess)

# global graph,model
# graph = tf.get_default_graph()
# sess = tf.Session(graph=graph)
# K.set_session(sess)


# 基本参数

class TotalLoss(Loss):
    """loss分两部分，一是seq2seq的交叉熵，二是相似度的交叉熵。
    """
    def compute_loss(self, inputs, mask=None):
        loss1 = self.compute_loss_of_seq2seq(inputs, mask)
        loss2 = self.compute_loss_of_similarity(inputs, mask)
        self.add_metric(loss1, name='seq2seq_loss')
        self.add_metric(loss2, name='similarity_loss')
        return loss1 + loss2

    def compute_loss_of_seq2seq(self, inputs, mask=None):
        y_true, y_mask, _, y_pred = inputs
        y_true = y_true[:, 1:]  # 目标token_ids
        y_mask = y_mask[:, 1:]  # segment_ids，刚好指示了要预测的部分
        y_pred = y_pred[:, :-1]  # 预测序列，错开一位
        loss = K.sparse_categorical_crossentropy(y_true, y_pred)
        loss = K.sum(loss * y_mask) / K.sum(y_mask)
        return loss

    def compute_loss_of_similarity(self, inputs, mask=None):
        _, _, y_pred, _ = inputs
        y_true = self.get_labels_of_similarity(y_pred)  # 构建标签
        y_pred = K.l2_normalize(y_pred, axis=1)  # 句向量归一化
        similarities = K.dot(y_pred, K.transpose(y_pred))  # 相似度矩阵
        similarities = similarities - K.eye(K.shape(y_pred)[0]) * 1e12  # 排除对角线
        similarities = similarities * 30  # scale
        loss = K.categorical_crossentropy(
            y_true, similarities, from_logits=True
        )
        return loss

    def get_labels_of_similarity(self, y_pred):
        idxs = K.arange(0, K.shape(y_pred)[0])
        idxs_1 = idxs[None, :]
        idxs_2 = (idxs + 1 - idxs % 2 * 2)[:, None]
        labels = K.equal(idxs_1, idxs_2)
        labels = K.cast(labels, K.floatx())
        return labels


class GenerateModel(object):
    def __init__(self):

        self.epoch_acc_vel = 0
        self.config_path = r'./chinese_roberta_wwm_ext_L-12_H-768_A-12/bert_config.json'
        self.checkpoint_path = r'./chinese_roberta_wwm_ext_L-12_H-768_A-12/bert_model.ckpt'
        self.dict_path = r'./chinese_roberta_wwm_ext_L-12_H-768_A-12/vocab.txt'
        self.maxlen = 120
        self.novel_maxlen = 60

    def device_setup(self):
        token_dict, keep_tokens = load_vocab(
            dict_path=self.dict_path,
            simplified=True,
            startswith=['[PAD]', '[UNK]', '[CLS]', '[SEP]'],
        )
        tokenizer = Tokenizer(token_dict, do_lower_case=True)

        # model = build_transformer_model(
        #     self.config_path,
        #     self.checkpoint_path,
        #     application='unilm',
        #     keep_tokens=keep_tokens,  # 只保留keep_tokens中的字，精简原字表
        # )

        bert = build_transformer_model(
            self.config_path,
            self.checkpoint_path,
            with_pool='linear',
            application='unilm',
            keep_tokens=keep_tokens,
            return_keras_model=False,
        )

        encoder = keras.models.Model(bert.model.inputs, bert.model.outputs[0])
        seq2seq = keras.models.Model(bert.model.inputs, bert.model.outputs[1])

        # output = CrossEntropy(2)(model.inputs + model.outputs)
        #
        # model = Model(model.inputs, output)
        # model = Model(model.inputs, model.outputs)

        outputs = TotalLoss([2, 3])(bert.model.inputs + bert.model.outputs)
        model = keras.models.Model(bert.model.inputs, outputs)

        path_model = './output_simbert_yy/best_simbertmodel_datasim.weights'
        model.load_weights(path_model)

        return encoder,seq2seq, tokenizer


class CrossEntropy(Loss):
    """交叉熵作为loss，并mask掉输入部分
    """
    def compute_loss(self, inputs, mask=None):
        y_true, y_mask, y_pred = inputs
        y_true = y_true[:, 1:]  # 目标token_ids
        y_mask = y_mask[:, 1:]  # segment_ids，刚好指示了要预测的部分
        y_pred = y_pred[:, :-1]  # 预测序列，错开一位
        loss = K.sparse_categorical_crossentropy(y_true, y_pred)
        loss = K.sum(loss * y_mask) / K.sum(y_mask)
        return loss

class AutoTitle(AutoRegressiveDecoder):
    """seq2seq解码器
    """
    def __init__(self, model, tokenizer, start_id, end_id, maxlen, minlen=1):
        super(AutoTitle, self).__init__(start_id, end_id, maxlen, minlen)
        self.model = model
        self.tokenizer = tokenizer
        self.start_id = start_id
        self.end_id = end_id
        self.minlen = minlen
        self.models = {}
        if start_id is None:
            self.first_output_ids = np.empty((1, 0), dtype=int)
        else:
            self.first_output_ids = np.array([[self.start_id]])

    def data_generator(self, inputs, output_ids):
        batch_token_ids, batch_segment_ids = [], []
        if output_ids == []:
            for txt in inputs:
                token_ids, segment_ids = self.tokenizer.encode(txt, maxlen=120)
                batch_token_ids.append(token_ids)
                batch_segment_ids.append(segment_ids)
        else:
            for txt,output_id in zip(inputs, output_ids):
                token_ids, segment_ids = self.tokenizer.encode(txt, output_id)
                batch_token_ids.append(token_ids[:-1])
                batch_segment_ids.append(segment_ids[:-1])

        batch_token_ids = sequence_padding(batch_token_ids)
        batch_segment_ids = sequence_padding(batch_segment_ids)
        return batch_token_ids, batch_segment_ids

    def beam_search_batch(self, inputs, topk, states=None, temperature=1, min_ends=1):
        """beam search解码
        说明：这里的topk即beam size；
        返回：最优解码序列。
        """
        inputs = [np.array([i]) for i in inputs]
        output_ids, output_scores = self.first_output_ids, np.zeros(1)
        for step in range(self.maxlen):
            scores, states = self.predict(
                inputs, output_ids, states, temperature, 'logits'
            )  # 计算当前得分
            if step == 0:  # 第1步预测后将输入重复topk次
                inputs = [np.repeat(i, topk, axis=0) for i in inputs]
            scores = output_scores.reshape((-1, 1)) + scores  # 综合累积得分
            indices = scores.argpartition(-topk, axis=None)[-topk:]  # 仅保留topk
            indices_1 = indices // scores.shape[1]  # 行索引
            indices_2 = (indices % scores.shape[1]).reshape((-1, 1))  # 列索引
            output_ids = np.concatenate([output_ids[indices_1], indices_2],
                                        1)  # 更新输出
            output_scores = np.take_along_axis(
                scores, indices, axis=None
            )  # 更新得分
            is_end = output_ids[:, -1] == self.end_id  # 标记是否以end标记结束
            end_counts = (output_ids == self.end_id).sum(1)  # 统计出现的end标记
            if output_ids.shape[1] >= self.minlen:  # 最短长度判断
                best = output_scores.argmax()  # 得分最大的那个
                if is_end[best] and end_counts[best] >= min_ends:  # 如果已经终止
                    return output_ids[best]  # 直接输出
                else:  # 否则，只保留未完成部分
                    flag = ~is_end | (end_counts < min_ends)  # 标记未完成序列
                    if not flag.all():  # 如果有已完成的
                        inputs = [i[flag] for i in inputs]  # 扔掉已完成序列
                        output_ids = output_ids[flag]  # 扔掉已完成序列
                        output_scores = output_scores[flag]  # 扔掉已完成序列
                        end_counts = end_counts[flag]  # 扔掉已完成end计数
                        topk = flag.sum()  # topk相应变化
        # 达到长度直接输出
        return output_ids[output_scores.argmax()]


    def random_sample_batch(
        self,
        inputs,
        n,
        topk=None,
        topp=None,
        states=None,
        temperature=1,
        min_ends=1
    ):
        """随机采样n个结果
        说明：非None的topk表示每一步只从概率最高的topk个中采样；而非None的topp
             表示每一步只从概率最高的且概率之和刚好达到topp的若干个token中采样。
        返回：n个解码序列组成的list。
        """
        inputs = [np.array([i for j in i]) for i in inputs]
        output_ids = self.first_output_ids
        results = []
        for step in range(self.maxlen):
            probas, states = self.predict(
                inputs, output_ids, states, temperature, 'probas'
            )  # 计算当前概率
            probas /= probas.sum(axis=1, keepdims=True)  # 确保归一化
            if step == 0:  # 第1步预测后将结果重复n次
                probas = np.repeat(probas, n, axis=0)
                inputs = [np.repeat(i, n, axis=0) for i in inputs]
                output_ids = np.repeat(output_ids, n, axis=0)
            if topk is not None:
                k_indices = probas.argpartition(-topk,
                                                axis=1)[:, -topk:]  # 仅保留topk
                probas = np.take_along_axis(probas, k_indices, axis=1)  # topk概率
                probas /= probas.sum(axis=1, keepdims=True)  # 重新归一化
            if topp is not None:
                p_indices = probas.argsort(axis=1)[:, ::-1]  # 从高到低排序
                probas = np.take_along_axis(probas, p_indices, axis=1)  # 排序概率
                cumsum_probas = np.cumsum(probas, axis=1)  # 累积概率
                flag = np.roll(cumsum_probas >= topp, 1, axis=1)  # 标记超过topp的部分
                flag[:, 0] = False  # 结合上面的np.roll，实现平移一位的效果
                probas[flag] = 0  # 后面的全部置零
                probas /= probas.sum(axis=1, keepdims=True)  # 重新归一化
            sample_func = lambda p: np.random.choice(len(p), p=p)  # 按概率采样函数
            sample_ids = np.apply_along_axis(sample_func, 1, probas)  # 执行采样
            sample_ids = sample_ids.reshape((-1, 1))  # 对齐形状
            if topp is not None:
                sample_ids = np.take_along_axis(
                    p_indices, sample_ids, axis=1
                )  # 对齐原id
            if topk is not None:
                sample_ids = np.take_along_axis(
                    k_indices, sample_ids, axis=1
                )  # 对齐原id
            output_ids = np.concatenate([output_ids, sample_ids], 1)  # 更新输出
            is_end = output_ids[:, -1] == self.end_id  # 标记是否以end标记结束
            end_counts = (output_ids == self.end_id).sum(1)  # 统计出现的end标记
            if output_ids.shape[1] >= self.minlen:  # 最短长度判断
                flag = is_end & (end_counts >= min_ends)  # 标记已完成序列
                if flag.any():  # 如果有已完成的
                    for ids in output_ids[flag]:  # 存好已完成序列
                        results.append(ids)
                    flag = (flag == False)  # 标记未完成序列
                    inputs = [i[flag] for i in inputs]  # 只保留未完成部分输入
                    output_ids = output_ids[flag]  # 只保留未完成部分候选集
                    end_counts = end_counts[flag]  # 只保留未完成部分end计数
                    if len(output_ids) == 0:
                        break
        # 如果还有未完成序列，直接放入结果
        for ids in output_ids:
            results.append(ids)
        # 返回结果
        return results

    def random_sample_and_beam_search(
        self,
        inputs,
        n,
        topk=None,
        topp=None,
        states=None,
        temperature=1,
        min_ends=1
    ):
        """随机采样n个结果
        说明：非None的topk表示每一步只从概率最高的topk个中采样；而非None的topp
             表示每一步只从概率最高的且概率之和刚好达到topp的若干个token中采样。
        返回：n个解码序列组成的list。
        """
        whether_end_b = False
        results_r = []
        results_b = []
        # index_r = [i for i in range(n)]
        # index_b = [i for i in range(topk)]
        index_r = np.arange(n)
        index_b = np.arange(topk)
        inputs = [np.array([i]) for i in inputs]
        output_ids, output_scores = self.first_output_ids, np.zeros(1)
        results = []
        for step in range(self.maxlen):
            beam_n = len(index_b)
            probas, states = self.predict(
                inputs, output_ids, states, temperature, 'probas'
            )  # 计算当前概率
            probas = probas / probas.sum(axis=1, keepdims=True)  # 确保归一化
            if step == 0:  # 第1步预测后将结果重复n次
                probas = np.repeat(probas, n + topk, axis=0)
                inputs_r = [np.repeat(i, n, axis=0) for i in inputs]
                output_ids = np.repeat(output_ids, n + topk, axis=0)
                inputs_b = [np.repeat(i, topk, axis=0) for i in inputs]
            else:
                if whether_end_b == False:
                    inputs_r = [i[:-beam_n, :] for i in inputs]
                    inputs_b = [i[-beam_n:, :] for i in inputs]
                else:
                    inputs_r = inputs
            if whether_end_b == False:
                probas_r = probas[:-beam_n, :]
            else:
                probas_r = probas
            if step == 0:
                probas_b = probas[0,:]
            else:
                probas_b = probas[-beam_n:, :]

            if whether_end_b == False:
                output_ids_r = output_ids[:-beam_n, :]
                output_ids_b = output_ids[-beam_n:, :]
            else:
                output_ids_r = output_ids
            k_indices = probas_r.argpartition(-topk,
                                            axis=1)[:, -topk:]  # 仅保留topk
            probas_r = np.take_along_axis(probas_r, k_indices, axis=1)  # topk概率
            probas_r /= probas_r.sum(axis=1, keepdims=True)  # 重新归一化

            if whether_end_b == False:
                scores = output_scores.reshape((-1, 1)) + probas_b  # 综合累积得分
                indices = scores.argpartition(-topk, axis=None)[-topk:]  # 仅保留topk
                indices_1 = indices // scores.shape[1]  # 行索引
                indices_2 = (indices % scores.shape[1]).reshape((-1, 1))  # 列索引
                try:
                    output_ids_b = np.concatenate([output_ids_b[indices_1], indices_2],
                                                1)  # 更新输出
                except:
                    print(output_ids_b.shape)
                    print(indices_1)
                    print(indices_2)
                    exit()
                output_scores = np.take_along_axis(
                    scores, indices, axis=None
                )  # 更新得分
            sample_func = lambda p: np.random.choice(len(p), p=p)  # 按概率采样函数
            try:
                sample_ids = np.apply_along_axis(sample_func, 1, probas_r)  # 执行采样
            except:
                print(probas_r)
            sample_ids = sample_ids.reshape((-1, 1))  # 对齐形状
            if topk is not None:
                sample_ids = np.take_along_axis(
                    k_indices, sample_ids, axis=1
                )  # 对齐原id
            output_ids_r = np.concatenate([output_ids_r, sample_ids], 1)  # 更新输出

            # output_ids = np.concatenate([output_ids_r, output_ids_b], 0)
            if whether_end_b == False:
                is_end_r = output_ids_r[:, -1] == self.end_id  # 标记是否以end标记结束
                is_end_b = output_ids_b[:, -1] == self.end_id  # 标记是否以end标记结束
            else:
                is_end_r = output_ids_r[:, -1] == self.end_id

            if whether_end_b == False:
                end_counts_r = (output_ids_r == self.end_id).sum(1)  # 统计出现的end标记
                end_counts_b = (output_ids_b == self.end_id).sum(1)  # 统计出现的end标记
            else:
                end_counts_r = (output_ids_r == self.end_id).sum(1)
            # random_serach
            if output_ids_r.shape[1] >= self.minlen:  # 最短长度判断
                flag = is_end_r & (end_counts_r >= min_ends)  # 标记已完成序列
                if flag.any():  # 如果有已完成的
                    for ids in output_ids_r[flag]:  # 存好已完成序列
                        results_r.append(ids)
                    flag = (flag == False)  # 标记未完成序列
                    try:
                        index_r = index_r[flag]
                    except:
                        print("flag",flag)
                        print("index_r",index_r)
                    inputs_r = [i[flag] for i in inputs_r]  # 只保留未完成部分输入
                    output_ids_r = output_ids_r[flag]  # 只保留未完成部分候选集
                    end_counts_r = end_counts_r[flag]  # 只保留未完成部分end计数

            # beam_serach
            if whether_end_b == False:
                if output_ids_b.shape[1] >= self.minlen:  # 最短长度判断
                    best = output_scores.argmax()  # 得分最大的那个
                    if is_end_b[best] and end_counts_b[best] >= min_ends:  # 如果已经终止
                        results_b.append(output_ids_b[best])  # 直接输出
                        whether_end_b = True
                    else:  # 否则，只保留未完成部分
                        flag_b = ~is_end_b | (end_counts_b < min_ends)  # 标记未完成序列
                        if not flag_b.all():  # 如果有已完成的
                            index_b = index_b[flag_b]
                            inputs_b = [i[flag_b] for i in inputs_b]  # 扔掉已完成序列
                            output_ids_b = output_ids_b[flag_b]  # 扔掉已完成序列
                            output_scores = output_scores[flag_b]  # 扔掉已完成序列
                            end_counts_b = end_counts_b[flag_b]  # 扔掉已完成end计数
                            topk = flag_b.sum()  # topk相应变化

            if whether_end_b == False and len(output_ids_r) != 0:
                token_r = inputs_r[0]
                sample_ids_r = inputs_r[1]
                token_b = inputs_b[0]
                sample_ids_b = inputs_b[1]
                token = np.concatenate([token_r,token_b],0)
                sample_ids = np.concatenate([sample_ids_r,sample_ids_b],0)
                inputs = [token,sample_ids]
                output_ids = np.concatenate([output_ids_r, output_ids_b], 0)
            elif whether_end_b == True and len(output_ids_r) != 0:
                inputs = inputs_r
                output_ids = output_ids_r
            elif whether_end_b == False and len(output_ids_r) == 0:
                inputs = inputs_b
                output_ids = output_ids_b
            else:
                break


        # 如果还有未完成序列，直接放入结果
        for ids in output_ids:
            results.append(ids)
        # 返回结果
        return results_r, results_b


    def top_batch(
        self,
        inputs_str,
        temperature=1,
        min_ends=1
    ):
        """随机采样n个结果
        说明：非None的topk表示每一步只从概率最高的topk个中采样；而非None的topp
             表示每一步只从概率最高的且概率之和刚好达到topp的若干个token中采样。
        返回：n个解码序列组成的list。
        """
        output_str = []
        # token_ids, segment_ids = self.data_generator(inputs, output_ids)
        batch_nums = len(inputs_str)
        output_ids =self.first_output_ids_batch = np.empty((batch_nums, 0), dtype=int)

        results = [[] for i in range(batch_nums)]
        index_data = [i for i in range(batch_nums)]
        for step in range(self.maxlen):

            token_ids, segment_ids = self.data_generator(inputs_str, output_str)
            inputs = [token_ids, segment_ids]
            probas = self.predict_batch(
                inputs
            )  # 计算当前概率
            # probas /= probas.sum(axis=1, keepdims=True)  # 确保归一化

            probas_new = []
            probas_bool = np.array(token_ids, dtype=bool)
            # np.array(np.where(probas_bool == True))
            for i,sentence in enumerate(probas_bool):
                lie = np.array(np.where(sentence == True))[0]
                probas_new.append(probas[i,lie[-1]])
            probas = np.array(probas_new)
            k_indices = np.argmax(probas,axis=1)  # 仅保留topk
            k_indices = k_indices.reshape(-1,1)

            sample_ids = k_indices
            output_ids = np.concatenate([output_ids, sample_ids], 1)  # 更新输出
            is_end = output_ids[:, -1] == self.end_id  # 标记是否以end标记结束
            end_counts = (output_ids == self.end_id).sum(1)  # 统计出现的end标记
            if output_ids.shape[1] >= self.minlen:  # 最短长度判断
                flag = is_end & (end_counts >= min_ends)  # 标记已完成序列
                if flag.any():  # 如果有已完成的
                    index = np.array(np.where(flag == True))[0]
                    pop_index = []
                    for i in index:
                        results[index_data[i]] = output_ids[i]
                        pop_index.append(index_data[i])
                    for i in pop_index:
                        index_data.remove(i)
                    # for ids in output_ids[flag]:  # 存好已完成序列
                    #     results.append(ids)
                    flag = (flag == False)  # 标记未完成序列
                    inputs_str = [inputs_str[i] for i in index_data]  # 只保留未完成部分输入
                    output_ids = output_ids[flag]  # 只保留未完成部分候选集
                    if len(output_ids) == 0:
                        break
                    else:
                        output_str = [tokenizer.decode(ids) for ids in output_ids]
                else:
                    output_str = [tokenizer.decode(ids) for ids in output_ids]
        # 如果还有未完成序列，直接放入结果
        # for ids in output_ids:
        #     results.append(ids)
        # 返回结果
        return results


    @AutoRegressiveDecoder.wraps(default_rtype='probas')
    def predict(self, inputs, output_ids, states):
        token_ids, segment_ids = inputs
        token_ids = np.concatenate([token_ids, output_ids], 1)
        segment_ids = np.concatenate([segment_ids, np.ones_like(output_ids)], 1)
        with graph.as_default():
            K.set_session(sess)
            nodes = self.last_token(self.model).predict([token_ids, segment_ids])
        return nodes
        # return self.last_token(self.model).predict([token_ids, segment_ids])

    def predict_batch(self, inputs):
        token_ids, segment_ids = inputs
        # token_ids = np.concatenate([token_ids, output_ids], 1)
        # segment_ids = np.concatenate([segment_ids, np.ones_like(output_ids)], 1)
        with graph.as_default():
            K.set_session(sess)
            nodes = self.model.predict([token_ids, segment_ids])
        return nodes
        # return self.last_token(self.model).predict([token_ids, segment_ids])

    def generate(self, text, topk=5):
        token_ids, segment_ids = self.tokenizer.encode(text, maxlen=256)
        output_ids = self.beam_search([token_ids, segment_ids],
                                      topk=topk)  # 基于beam search
        return self.tokenizer.decode(output_ids)

    def generate_random(self, text, n=20, topk=5):
        if isinstance(text, list):
            text = text[0]
        token_ids, segment_ids = self.tokenizer.encode(text, maxlen=120)
        output_ids = self.random_sample([token_ids, segment_ids], n, topk)  # 基于随机采样
        return [tokenizer.decode(ids) for ids in output_ids]

    def generate_random_topp(self, text, n=20, topp=0.98):
        if isinstance(text, list):
            text = text[0]
        token_ids, segment_ids = self.tokenizer.encode(text, maxlen=120)
        output_ids = self.random_sample([token_ids, segment_ids], n, topp=topp)  # 基于随机采样
        return [tokenizer.decode(ids) for ids in output_ids]

    def generate_top(self, text):
        output_ids = self.top_batch(text)  # 基于随机采样
        return [tokenizer.decode(ids) for ids in output_ids]

    def generate_random_sample_and_beam_search(self, text, n=20, topk=5):
        text = text[0]
        token_ids, segment_ids = self.tokenizer.encode(text, maxlen=120)
        output_ids_r, output_ids_b = self.random_sample_and_beam_search([token_ids, segment_ids], n=n,
                                      topk=topk)  # 基于beam search
        output_str_r = [self.tokenizer.decode(ids) for ids in output_ids_r]
        output_str_b = [self.tokenizer.decode(ids) for ids in output_ids_b]
        return output_str_r, output_str_b

    def gen_synonyms(self, text, n=20):
        """"含义： 产生sent的n个相似句，然后返回最相似的k个。
        做法：用seq2seq生成，并用encoder算相似度并排序。
        """
        r = self.generate_random_topp(text, n)
        r = [i for i in set(r) if i != text]
        r = [text] + r
        X, S = [], []
        for t in r:
            x, s = tokenizer.encode(t)
            X.append(x)
            S.append(s)
        X = sequence_padding(X)
        S = sequence_padding(S)
        Z = encoder.predict([X, S])
        Z /= (Z ** 2).sum(axis=1, keepdims=True) ** 0.5
        argsort = np.dot(Z[1:], -Z[0]).argsort()
        return [r[i + 1] for i in argsort]


    def gen_synonyms_short(self, text, n=20, len_s = 0.9):
        """"含义： 产生sent的n个相似句，然后返回最相似的k个。
        做法：用seq2seq生成，并用encoder算相似度并排序。
        """
        if isinstance(text, list):
            text = text[0]
        new_text_len = int(len(text) * len_s)
        r = self.generate_random(text, n)
        r = [i for i in set(r) if i != text]
        r = [text] + r
        X, S = [], []
        for t in r:
            x, s = tokenizer.encode(t)
            X.append(x)
            S.append(s)
        X = sequence_padding(X)
        S = sequence_padding(S)
        with graph.as_default():
            K.set_session(sess)
            Z = encoder.predict([X, S])
        Z /= (Z ** 2).sum(axis=1, keepdims=True) ** 0.5
        argsort = np.dot(Z[1:], -Z[0]).argsort()
        sentence_list = [r[i + 1] for i in argsort]

        return_list = []
        for i in sentence_list:
            if len(i) < new_text_len:
                return_list.append(i)
                break

        for i in sentence_list:
            if new_text_len <len(i) < len(text):
                return_list.append(i)
                break
        if return_list != []:
            return return_list[0]
        else:
            return sentence_list[0]

generatemodel = GenerateModel()
encoder,seq2seq, tokenizer = generatemodel.device_setup()
autotitle = AutoTitle(seq2seq, tokenizer, start_id=None, end_id=tokenizer._token_end_id, maxlen=120)




def just_show(file):
    data = []
    try:
        with open(file, 'r', encoding="utf-8") as f:
            lines = [x.strip() for x in f if x.strip() != '']
    except:
        with open(file, 'r', encoding="gbk") as f:
            lines = [x.strip() for x in f if x.strip() != '']
    # s2 = u'她只能应下来。'
    # lines = pd.read_csv(file,encoding="gbk").values.tolist()
    # random.shuffle(lines)
    # lines = lines[:20]
    for s in tqdm(lines[:2]):
        print(s)
        pre = autotitle.generate_random(s)
        print(s)
        print(pre)
        # data.append([s, pre])
    # pd.DataFrame(data,columns=["原始文本","生成文本"]).to_csv("data/text_测试一万字_unilm_修正数据_小说预训练_全部数据_epoch72_反向训练.csv")


def just_show_sentence(file: list) -> object:
    """
    @param file:list
    """
    text = file[0]
    pre = autotitle.generate(text)
    return pre

def just_show_sentence_batch(file: list) -> object:
    text = file
    pre = autotitle.generate_top(text)
    return pre

def just_show_csv_random(file):
    data_new = []
    data = pd.read_csv(file).values.tolist()
    for sentence in tqdm(data):
        sentence = sentence[1]
        print(sentence)
        data_new_dan = []
        data_new_dan.extend([sentence, len(sentence)])
        pre = autotitle.generate_random(sentence)
        for i in pre:
            data_new_dan.extend([i, len(i)])

        data_new.append(data_new_dan)
    pd.DataFrame(data_new).to_csv("data/###第3章 非常尴尬_sim_topK_5.csv")
    # return pre


def just_show_chachong_random(file):
    text = file[0]
    pre = autotitle.gen_synonyms(text)
    return pre


def just_show_csv_beam(file):
    data_new = []
    data = pd.read_csv(file).values.tolist()
    for sentence in tqdm(data):
        sentence = sentence[1]
        print(sentence)
        data_new_dan = []
        data_new_dan.extend([sentence, len(sentence)])
        pre = autotitle.generate([sentence])
        print(pre)
        data_new_dan.extend([pre, len(pre)])
        data_new.append(data_new_dan)
    pd.DataFrame(data_new).to_csv("data/###第3章 非常尴尬_sim_topK_1.csv")



if __name__ == '__main__':

    # text = ["强调轻资产经营, 更加重视经营风险的规避", "历史和当下都证明,创新是民族生存、发展的不竭源泉,是是自身发展的必然选择", "是时代对于青年们的深切呼唤"]
    # # print(just_show_sentence_batch(text))
    # # print(type(just_show_sentence_batch(text)))
    # print(just_show_chachong_random(text))

    # path = './data/11篇excel'
    # path_exist = "./data/11篇临时拼接2"
    # path_list = []
    # for file_name in os.listdir(path):
    #     path_list.append(file_name)
    #
    # path_list_exist = []
    # for file_name in os.listdir(path_exist):
    #     file_name_0 = file_name.split(".")[0]
    #     file_name_1 = file_name.split(".")[1]
    #     file_name = file_name_0[:-1] + "." + file_name_1
    #     path_list_exist.append(file_name)
    #
    # for docx_name in path_list:
    #     if docx_name in path_list_exist:
    #         continue
    #     df_list_new = []
    #     df = pd.read_excel(path + "/" + docx_name).values.tolist()
    #     for dan in tqdm(df):
    #         pre = just_show_sentence([dan[1]])
    #         df_list_new.append([dan[0], pre])
    #     df = pd.DataFrame(df_list_new)
    #     file_name = docx_name.split(".")[0]
    #     df.to_excel("./data/11篇临时拼接2/{}_.xlsx".format(file_name), index=None)
    path = './data/11篇model1'
    path_new = './data/11篇model3'
    path_list = []
    for file_name in os.listdir(path):
        path_list.append(file_name)
    for docx_name in path_list:
        df_list_new = []
        with open(path + "/" + docx_name, 'r', encoding="utf-8") as f:
            lines = [x.strip() for x in f if x.strip() != '']
        for dan in tqdm(lines):
            break_ = False
            for i in dan:
                if i == "章":
                    break_ = True
                    break
            if break_ == True:
                df_list_new.append(dan)
                continue
            pre = just_show_sentence([dan])
            df_list_new.append(pre)

        with open(path_new + "/" + docx_name, "w", encoding='utf-8') as file:
            for i in df_list_new:
                file.write(i + '\n')
            file.close()