drop_weight_rewrite/data_do/筛选训练数据层级细分strsim.py

# -*- coding: utf-8 -*-

"""
@Time    :  2023/1/31 19:02
@Author  :
@FileName:
@Software:
@Describe:
"""
import os
# os.environ["TF_KERAS"] = "1"
import pandas as pd

os.environ["CUDA_VISIBLE_DEVICES"] = "0"
import json
import numpy as np
from bert4keras.backend import keras, set_gelu
from bert4keras.tokenizers import Tokenizer, load_vocab
from bert4keras.models import build_transformer_model
from bert4keras.optimizers import Adam, extend_with_piecewise_linear_lr
from bert4keras.snippets import sequence_padding, DataGenerator
from bert4keras.snippets import open
from keras.layers import Lambda, Dense
import tensorflow as tf
from keras.backend import set_session
from sklearn.metrics.pairwise import cosine_similarity
from rouge import Rouge  # pip install rouge
from nltk.translate.bleu_score import sentence_bleu, SmoothingFunction
from tqdm import tqdm
import jieba
from gensim.models import KeyedVectors, word2vec, Word2Vec
import random
import difflib

config = tf.ConfigProto()
config.gpu_options.allow_growth = True
set_session(tf.Session(config=config)) # 此处不同

class Word2vecModel:
    def __init__(self):
        self.path = "E:\pycharm_workspace\查重分析\word2vec_model\\word2vec_add_new_18.model"
        self.model = Word2Vec.load(self.path)

    def word2vec_res(self,seg_0_list, seg_1_list):
        sentence_0_list = []
        sentence_1_list = []
        for i in seg_0_list:
            a = self.model.wv[i]
            sentence_0_list.append(a)

        for i in seg_1_list:
            a = self.model.wv[i]
            sentence_1_list.append(a)

        return sentence_0_list, sentence_1_list

class Evaluator(keras.callbacks.Callback):
    """评估与保存
    """

    def __init__(self):
        self.rouge = Rouge()
        self.smooth = SmoothingFunction().method1
        self.best_bleu = 0.

    # def on_epoch_end(self, epoch, logs=None):
    #     metrics = self.evaluate(valid_data)  # 评测模型
    #     if metrics['bleu'] > self.best_bleu:
    #         self.best_bleu = metrics['bleu']
    #         model.save_weights('./best_model.weights')  # 保存模型
    #     metrics['best_bleu'] = self.best_bleu
    #     print('valid_data:', metrics)


    def evaluate_t(self, data_1, data_2, topk=1):
        total = 0
        rouge_1, rouge_2, rouge_l, bleu = 0, 0, 0, 0

        scores = self.rouge.get_scores(hyps=[data_1], refs=[data_2])
        rouge_1 += scores[0]['rouge-1']['f']
        rouge_2 += scores[0]['rouge-2']['f']
        rouge_l += scores[0]['rouge-l']['f']
        bleu += sentence_bleu(
            references=[data_1.split(' ')],
            hypothesis=data_2.split(' '),
            smoothing_function=self.smooth
        )
        # rouge_1 /= total
        # rouge_2 /= total
        # rouge_l /= total
        # bleu /= total
        return [rouge_1, rouge_2, rouge_l, bleu]

class bertModel:
    def __init__(self):
        self.config_path = '../chinese_roberta_wwm_ext_L-12_H-768_A-12/bert_config.json'
        self.checkpoint_path = '../chinese_roberta_wwm_ext_L-12_H-768_A-12/bert_model.ckpt'
        self.dict_path = '../chinese_roberta_wwm_ext_L-12_H-768_A-12/vocab.txt'
        self.token_dict, self.keep_tokens = load_vocab(
            dict_path=self.dict_path,
            simplified=True,
            startswith=['[PAD]', '[UNK]', '[CLS]', '[SEP]'],
        )
        self.tokenizer = Tokenizer(self.token_dict, do_lower_case=True)
        self.buildmodel()


    def buildmodel(self):
        bert = build_transformer_model(
            config_path=self.config_path,
            checkpoint_path=self.checkpoint_path,
            return_keras_model=False,
        )

        output = Lambda(lambda x: x[:, 0], name='CLS-token')(bert.model.output)
        self.model = keras.models.Model(bert.model.input, output)
        self.model.summary()

    def predict(self,text):
        batch_token_ids, batch_segment_ids = [], []
        token_ids, segment_ids = self.tokenizer.encode(text, maxlen=256)
        batch_token_ids.append(token_ids)
        batch_segment_ids.append(segment_ids)
        return self.model.predict([batch_token_ids, batch_segment_ids])


def simbert(data_1, data_2):
    pass

def word2vec():
    pass

def bleu():
    pass


if __name__ == '__main__':
    file = "../data/train_yy_zong_sim_99.txt"
    sim_value = [1, 0.95, 0.9, 0.85, 0.8, 0.75, 0.7, 0]
    model = bertModel()
    eval_class = Evaluator()
    # word2vecmodel = Word2vecModel()
    try:
        with open(file, 'r', encoding="utf-8") as f:
            lines = [x.strip() for x in f if x.strip() != '']
    except:
        with open(file, 'r', encoding="gbk") as f:
            lines = [x.strip() for x in f if x.strip() != '']

    bertsim_list = []
    bleusim_list = []
    word2vecsim_list = []
    data_train_text = []

    random.shuffle(lines)
    print(len(lines))
    for txt in tqdm(lines):
        text = txt.split('\t')
        if len(text) == 3:
            data_1 = text[0]
            data_2 = text[2]
            str_sim_value = difflib.SequenceMatcher(None, data_1, data_2).quick_ratio()
            # if len(data_2) - len(data_1) < 0 and len(data_2) / len(data_1) > 0.8:
            #     num_yu = 1 - len(data_2) / len(data_1)
            #     str_sim_value = 1 - str_sim_value * num_yu

            if 1 >= str_sim_value > 0.95:
                data_train_text.append([data_1, data_2, str(str_sim_value), "1-0.95"])
            elif 0.95 >= str_sim_value > 0.9:
                data_train_text.append([data_1, data_2, str(str_sim_value), "0.95-0.9"])
            elif 0.9 >= str_sim_value > 0.85:
                data_train_text.append([data_1, data_2, str(str_sim_value), "0.9-0.85"])
            elif 0.85 >= str_sim_value > 0.8:
                data_train_text.append([data_1, data_2, str(str_sim_value), "0.85-0.8"])
            elif 0.8 >= str_sim_value > 0.75:
                data_train_text.append([data_1, data_2, str(str_sim_value), "0.8-0.75"])
            elif 0.75 >= str_sim_value > 0.7:
                data_train_text.append([data_1, data_2, str(str_sim_value), "0.75-0.7"])
            else:
                data_train_text.append([data_1, data_2, str(str_sim_value), "0.7 - 0"])

    data_train_text = sorted(data_train_text, key=lambda x:x[2], reverse=True)
    df = pd.DataFrame(data_train_text)
    print(df)
    df.to_csv("../data/yy改写相似度_1.csv", index=None)
    df.to_excel("../data/yy改写相似度_1.xlsx", index=None)