|
|
|
@ -1,8 +1,10 @@ |
|
|
|
# -*- coding: utf-8 -*- |
|
|
|
|
|
|
|
""" |
|
|
|
@Time : 2022/8/15 15:20 |
|
|
|
@Author : |
|
|
|
@FileName: |
|
|
|
@Software: |
|
|
|
@Author : |
|
|
|
@FileName: |
|
|
|
@Software: |
|
|
|
@Describe: |
|
|
|
""" |
|
|
|
import json |
|
|
|
@ -19,13 +21,11 @@ from bert4keras.snippets import DataGenerator, AutoRegressiveDecoder |
|
|
|
from keras.models import Model |
|
|
|
from rouge import Rouge # pip install rouge |
|
|
|
from nltk.translate.bleu_score import sentence_bleu, SmoothingFunction |
|
|
|
import difflib |
|
|
|
|
|
|
|
|
|
|
|
class Evaluator(keras.callbacks.Callback): |
|
|
|
"""评估与保存 |
|
|
|
""" |
|
|
|
|
|
|
|
def __init__(self): |
|
|
|
self.rouge = Rouge() |
|
|
|
self.smooth = SmoothingFunction().method1 |
|
|
|
@ -69,79 +69,52 @@ class Evaluator(keras.callbacks.Callback): |
|
|
|
} |
|
|
|
|
|
|
|
def evaluate_t(self, data_1, data_2, topk=1): |
|
|
|
|
|
|
|
data_1_eval = ' '.join(data_1) |
|
|
|
data_2_eval = ' '.join(data_2) |
|
|
|
total = 0 |
|
|
|
rouge_1, rouge_2, rouge_l, bleu = 0, 0, 0, 0 |
|
|
|
|
|
|
|
scores = self.rouge.get_scores(hyps=[data_1_eval], refs=[data_2_eval]) |
|
|
|
scores = self.rouge.get_scores(hyps=[data_1], refs=[data_2]) |
|
|
|
rouge_1 += scores[0]['rouge-1']['f'] |
|
|
|
rouge_2 += scores[0]['rouge-2']['f'] |
|
|
|
rouge_l += scores[0]['rouge-l']['f'] |
|
|
|
bleu += sentence_bleu( |
|
|
|
references=[data_1_eval.split(' ')], |
|
|
|
hypothesis=data_2_eval.split(' '), |
|
|
|
references=[data_1.split(' ')], |
|
|
|
hypothesis=data_2.split(' '), |
|
|
|
smoothing_function=self.smooth |
|
|
|
) |
|
|
|
# rouge_1 /= total |
|
|
|
# rouge_2 /= total |
|
|
|
# rouge_l /= total |
|
|
|
# bleu /= total |
|
|
|
str_sim = difflib.SequenceMatcher(None, data_1, data_2).quick_ratio() |
|
|
|
return [rouge_1, rouge_2, rouge_l, bleu, str_sim] |
|
|
|
return [rouge_1, rouge_2, rouge_l, bleu] |
|
|
|
|
|
|
|
|
|
|
|
eval_class = Evaluator() |
|
|
|
|
|
|
|
# print(eval_class.evaluate_t("星 辰 的 话","星 辰 的 话 :")) |
|
|
|
path = "data/700条效果对比.xlsx" |
|
|
|
path_out = "data/700条效果对比测评结果_14.csv" |
|
|
|
path = "data/一万字小说测试效果.xlsx" |
|
|
|
path_out = "data/一万字小说测试效果测评.csv" |
|
|
|
data = pd.read_excel(path).values.tolist() |
|
|
|
data_new = {"rouge_1": [0,0,0,0,0,0,0,0,0,0,0], |
|
|
|
"rouge_2": [0,0,0,0,0,0,0,0,0,0,0], |
|
|
|
"rouge_l": [0,0,0,0,0,0,0,0,0,0,0], |
|
|
|
"bleu": [0,0,0,0,0,0,0,0,0,0,0]} |
|
|
|
total = 0 |
|
|
|
|
|
|
|
list_class = [0 for i in range(13)] |
|
|
|
# print(list_class) |
|
|
|
data_new = {"rouge_1": list_class.copy(), |
|
|
|
"rouge_2": list_class.copy(), |
|
|
|
"rouge_l": list_class.copy(), |
|
|
|
"bleu": list_class.copy(), |
|
|
|
"str_sim": list_class.copy()} |
|
|
|
total = len(data) |
|
|
|
|
|
|
|
print(len(data)) |
|
|
|
for i in data: |
|
|
|
dan_list = [i[1], i[2], i[3], i[4], i[5], i[6], i[7], i[8], i[9], i[10], i[11], i[12], i[-1]] |
|
|
|
dan_list = i[2:-1] |
|
|
|
for j in range(len(dan_list)): |
|
|
|
eval_list = eval_class.evaluate_t(dan_list[j], i[0]) |
|
|
|
try: |
|
|
|
data_new["rouge_1"][j] += eval_list[0] |
|
|
|
data_new["rouge_2"][j] += eval_list[1] |
|
|
|
data_new["rouge_l"][j] += eval_list[2] |
|
|
|
data_new["bleu"][j] += eval_list[3] |
|
|
|
data_new["str_sim"][j] += eval_list[4] |
|
|
|
except: |
|
|
|
pass |
|
|
|
eval_list = eval_class.evaluate_t(' '.join(dan_list[j]), ' '.join(i[-1])) |
|
|
|
data_new["rouge_1"][j] += eval_list[0] |
|
|
|
data_new["rouge_2"][j] += eval_list[1] |
|
|
|
data_new["rouge_l"][j] += eval_list[2] |
|
|
|
data_new["bleu"][j] += eval_list[3] |
|
|
|
|
|
|
|
data = {} |
|
|
|
''' |
|
|
|
生成文本（t5_未修正数据） 生成文本（unilm未修正数据） 生成文本（unilm修正数据） 生成文本（unilm修正数据_预训练） 生成文本（240w/24H） 生成文本(240W/48H) 生成文本(240W/24H/修) 生成文本（全部数据/72H/修) 生成文本（全部数据/72H/未修） 生成文本（t5修正数据） 生成文本（t5修正数据_190epoch） |
|
|
|
|
|
|
|
def fune(x): |
|
|
|
return x/total |
|
|
|
for i in data_new: |
|
|
|
data[i] = list(map(fune, data_new[i])) |
|
|
|
''' |
|
|
|
|
|
|
|
pd.DataFrame(data, |
|
|
|
index=["simbert_5day", |
|
|
|
"simbert_simdata4day", |
|
|
|
"simbert_simdata5day", |
|
|
|
"simbert_random20_5day", |
|
|
|
"simbert_simdata4day_yinhao", |
|
|
|
"simbert_simdata4day_yinhao_dropout", |
|
|
|
"simsim模型", |
|
|
|
"dropout_sim_03模型", |
|
|
|
"dropout_sim_04模型", |
|
|
|
"t5", |
|
|
|
"t5_dropout", |
|
|
|
"小说模型", |
|
|
|
"yy"] |
|
|
|
).to_csv( |
|
|
|
path_out) |
|
|
|
pd.DataFrame(data_new,index=["生成文本（t5_未修正数据）","生成文本（unilm未修正数据）","生成文本（unilm修正数据）", |
|
|
|
"生成文本（unilm修正数据_预训练）","生成文本（240w/24H）","生成文本(240W/48H)","生成文本(240W/24H/修)", |
|
|
|
"生成文本（全部数据/72H/修)", "生成文本（全部数据/72H/未修）", "生成文本（t5修正数据）", "生成文本（t5修正数据_190epoch）"]).to_csv(path_out) |
majiahui@haimaqingfan.com
8 months ago