You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
1590 lines
56 KiB
1590 lines
56 KiB
# -*- coding:utf-8 -*-
|
|
import os
|
|
import numpy as np
|
|
from numpy.linalg import norm
|
|
import pandas as pd
|
|
# from rouge import Rouge
|
|
from rouge_chinese import Rouge
|
|
from Rouge_w import Rouge_w, Rouge_l
|
|
import json
|
|
# import pymysql
|
|
import re
|
|
import requests
|
|
from flask import Flask, jsonify
|
|
from flask import request
|
|
import uuid
|
|
import time
|
|
import redis
|
|
from threading import Thread
|
|
from multiprocessing import Pool
|
|
|
|
app = Flask(__name__)
|
|
app.config["JSON_AS_ASCII"] = False
|
|
|
|
# pool = redis.ConnectionPool(host='localhost', port=63179, max_connections=100, db=16, password="zhicheng123*")
|
|
|
|
pool = redis.ConnectionPool(host='192.168.31.74', port=63179, max_connections=100, db=17, password="zhicheng123*")
|
|
redis_ = redis.Redis(connection_pool=pool, decode_responses=True)
|
|
|
|
db_key_querying = 'querying_check_task'
|
|
db_key_queryset = 'queryset_check_task'
|
|
db_key_query_recall = 'query_recall'
|
|
|
|
|
|
nums_cpus = 24
|
|
rouge = Rouge()
|
|
rouge_model = Rouge_w()
|
|
rouge_l_model = Rouge_l()
|
|
|
|
def jaccard_similarity(s1, s2):
|
|
set1 = set(s1)
|
|
set2 = set(s2)
|
|
intersection = set1 & set2
|
|
union = set1 | set2
|
|
return len(intersection) / len(union)
|
|
|
|
# def bert_check(text, recall_data_list):
|
|
# '''
|
|
# bert 查重
|
|
# :return:
|
|
# '''
|
|
#
|
|
# sen_0 = [text] * len(recall_data_list)
|
|
# sen_1 = [i[0] for i in recall_data_list]
|
|
#
|
|
# return_list = []
|
|
# request_json = {
|
|
# "texts": [sen_0, sen_1],
|
|
# }
|
|
# paper_dict = dialog_line_parse("http://192.168.31.74:16002/", request_json)
|
|
# score_list = paper_dict["res"]
|
|
#
|
|
# # 后期要改
|
|
# # return_list.append(re1[0][1])
|
|
# # return_list.append(re1[0][0])
|
|
# if 1 in score_list:
|
|
# index_score = score_list.index(1)
|
|
# else:
|
|
# index_score = "NaN"
|
|
#
|
|
# if index_score == "NaN":
|
|
# return_list.append(0)
|
|
# return_list.append("")
|
|
# else:
|
|
# return_list.append(1)
|
|
# return_list.append(index_score)
|
|
#
|
|
# return return_list
|
|
|
|
|
|
def rouge_value_self(data_1, data_2):
|
|
data_1 = [' '.join(i) for i in data_1]
|
|
data_2 = [' '.join(i) for i in data_2]
|
|
rouge_l_list = []
|
|
|
|
for sen_1, sen_2 in zip(data_1, data_2):
|
|
sen_1 = sen_1.split(" ")
|
|
sen_2 = sen_2.split(" ")
|
|
rouge_l_score = rouge_l_model.score(sen_1, sen_2)
|
|
rouge_l_list.append(rouge_l_score)
|
|
|
|
return "", "", rouge_l_list
|
|
|
|
|
|
def strsim_value(data_1, data_2):
|
|
data_1 = [' '.join(i) for i in data_1]
|
|
data_2 = [' '.join(i) for i in data_2]
|
|
rouge_l_list = []
|
|
|
|
for sen_1, sen_2 in zip(data_1, data_2):
|
|
sen_1 = sen_1.split(" ")
|
|
sen_2 = sen_2.split(" ")
|
|
rouge_l_score = jaccard_similarity(sen_1, sen_2)
|
|
rouge_l_list.append(rouge_l_score)
|
|
|
|
return "", "", rouge_l_list
|
|
|
|
|
|
def rouge_pre(text, df_train_nuoche):
|
|
return_list = []
|
|
index_rouge_list = []
|
|
text_list = [text] * len(df_train_nuoche)
|
|
|
|
data_list = []
|
|
for data_dan in df_train_nuoche:
|
|
data_list.append(data_dan[0])
|
|
rouge_1, rouge_2, rouge_l = rouge_value_self(text_list, data_list)
|
|
index_rouge_list.extend(rouge_l)
|
|
|
|
re1 = [(i[0], i[1]) for i in sorted(list(enumerate(index_rouge_list)), key=lambda x: x[1], reverse=True)]
|
|
|
|
return_list.append(re1[0][1])
|
|
return_list.append(re1[0][0])
|
|
|
|
return return_list
|
|
|
|
|
|
def rouge_pre_m(text, df_train_nuoche):
|
|
|
|
return_list = []
|
|
index_rouge_list = []
|
|
|
|
text_list = [text] * len(df_train_nuoche)
|
|
|
|
data_list = []
|
|
for data_dan in df_train_nuoche:
|
|
data_list.append(data_dan[0])
|
|
rouge_1, rouge_2, rouge_l = strsim_value(text_list, data_list)
|
|
index_rouge_list.extend(rouge_l)
|
|
|
|
re1 = [(i[0], i[1]) for i in sorted(list(enumerate(index_rouge_list)), key=lambda x: x[1], reverse=True)]
|
|
|
|
return_list.extend(re1)
|
|
|
|
return return_list
|
|
|
|
|
|
def rouge_pre_m_1(bool_check_sentense, content_list, recall_data_list):
|
|
# bool_check_sentense [[[1, 223],[1, 226], [1, 562]],[[2, 243],[2, 226], [2, 561]]]
|
|
|
|
bool_check_sentense_new = []
|
|
for bool_check_sentense_dan in bool_check_sentense:
|
|
bool_check_sentense_new_dan = []
|
|
|
|
text_list = []
|
|
data_list = []
|
|
linshi = []
|
|
for i in bool_check_sentense_dan:
|
|
text1 = content_list[i[0]]
|
|
text2 = recall_data_list[i[1]][0]
|
|
linshi.append([i[0], i[1]])
|
|
text_list.append(text1)
|
|
data_list.append(text2)
|
|
_, _, rouge_l_list = rouge_value_self(text_list, data_list)
|
|
for i in range(len(rouge_l_list)):
|
|
if rouge_l_list[i] > 0.47:
|
|
bool_check_sentense_new_dan.append(linshi[i])
|
|
if bool_check_sentense_new_dan != []:
|
|
bool_check_sentense_new.append(bool_check_sentense_new_dan)
|
|
return bool_check_sentense_new
|
|
|
|
# 以单个章节为例
|
|
def similar_content_func():
|
|
'''
|
|
重复文章
|
|
:return:
|
|
'''
|
|
return [{
|
|
"content": "重复的内容标红",
|
|
"thesis_info": "论文标题 + 论文作者 + 来源 + 年份日期--敞开式奶牛舍环境控制系统的设计 李晓红,刘晓丽,余泳昌 - 商丘工学院机械工程学院 - 2015-04-01",
|
|
"title": "标题",
|
|
"year": "日期",
|
|
"degree": "来源",
|
|
"author": "作者"
|
|
}]
|
|
|
|
|
|
def original_text_contrast_func(data_sentence_dan, paper_dict, content_list):
|
|
'''
|
|
重复的对比详细信息
|
|
:param similar_content:
|
|
:return:
|
|
'''
|
|
|
|
if data_sentence_dan != []:
|
|
original_text = ""
|
|
start = len(data_sentence_dan[0][1])
|
|
end = 0
|
|
similar_content = []
|
|
for i in data_sentence_dan: # 可能有很多个暂且确定是一个
|
|
|
|
similar_content_dan = {
|
|
"paper_red_len_word": "",
|
|
"content": "重复的内容标红",
|
|
"thesis_info": "论文标题 + 论文作者 + 来源 + 年份日期--敞开式奶牛舍环境控制系统的设计 李晓红,刘晓丽,余泳昌 - 商丘工学院机械工程学院 - 2015-04-01",
|
|
"title": "标题",
|
|
"year": "日期",
|
|
"degree": "来源",
|
|
"author": "作者",
|
|
"paper_len_word": ""
|
|
}
|
|
|
|
sentence_0_bool, sentence_0_dan_red = original_text_marked_red(i[1], paper_dict[i[0]][0],
|
|
paper_dict[i[0]][4][0],
|
|
paper_dict[i[0]][4][1]) # text_original, bert_text, bert_text_pre
|
|
|
|
sentence_1_bool, sentence_1_dan_red = original_text_marked_red(i[2], paper_dict[i[0]][2],
|
|
paper_dict[i[0]][4][2],
|
|
paper_dict[i[0]][4][3]) # text_original, bert_text, bert_text_pre
|
|
|
|
if sentence_0_bool == False or sentence_1_bool == False:
|
|
continue
|
|
|
|
start_dan = sentence_0_dan_red.index("<red>")
|
|
end_dan = sentence_0_dan_red.index("</red>") - len("<red>")
|
|
|
|
if start_dan < start:
|
|
start = start_dan
|
|
if end_dan > end:
|
|
end = end_dan
|
|
|
|
similar_content_dan["content"] = sentence_1_dan_red
|
|
similar_content_dan["title"] = i[3]
|
|
similar_content_dan["author"] = ""
|
|
similar_content_dan["degree"] = ""
|
|
similar_content_dan["year"] = ""
|
|
similar_content_dan["paper_len_word"] = ""
|
|
similar_content_dan["paper_red_len_word"] = end_dan - start_dan
|
|
|
|
thesis_info = " ".join(
|
|
[similar_content_dan["title"], similar_content_dan["author"], similar_content_dan["degree"],
|
|
similar_content_dan["year"]])
|
|
similar_content_dan["thesis_info"] = thesis_info
|
|
|
|
similar_content.append(similar_content_dan)
|
|
|
|
original_text_list = list(data_sentence_dan[0][1])
|
|
# original_text_list.insert(end, "</red>\n")
|
|
# original_text_list.insert(start, "\n<red>")
|
|
target_text_str = "".join(["\n<red>"] + original_text_list[start: end] + ["</red>\n"])
|
|
|
|
original_text_start = "".join(original_text_list[:start])
|
|
original_text_end = "".join(original_text_list[end:])
|
|
|
|
print(data_sentence_dan)
|
|
if data_sentence_dan[0][4][0]-1 < 0:
|
|
start_sen = ""
|
|
else:
|
|
start_sen = content_list[data_sentence_dan[0][4][0]-1]
|
|
|
|
if data_sentence_dan[0][4][-1]+1 >= len(content_list):
|
|
end_sen = ""
|
|
else:
|
|
end_sen = content_list[data_sentence_dan[0][4][-1]+1]
|
|
|
|
start_sen = start_sen + original_text_start
|
|
end_sen = original_text_end + end_sen
|
|
original_text = "此处有 {} 字相似\n".format(str(end - start)) + start_sen[-60:] + target_text_str + end_sen[:60]
|
|
else:
|
|
original_text = ""
|
|
end = 0
|
|
start = 0
|
|
similar_content = []
|
|
return_info = {
|
|
"original_text": original_text,
|
|
"dan_sentence_word_nums": end - start,
|
|
"similar_content": similar_content
|
|
}
|
|
return return_info
|
|
|
|
|
|
def repeat_quote_info_func(original_text_contrast, section_words):
|
|
'''
|
|
重复的引用信息
|
|
:return:
|
|
'''
|
|
chongfuwendang = {}
|
|
|
|
for sentence_dan in original_text_contrast:
|
|
for i in sentence_dan["similar_content"]:
|
|
thesis_info = i["thesis_info"]
|
|
if thesis_info not in chongfuwendang:
|
|
chongfuwendang[thesis_info] = {
|
|
"quote": False,
|
|
"thesis_author": i["author"],
|
|
"thesis_date": i["year"],
|
|
"thesis_info": thesis_info,
|
|
"thesis_repeat_rate": (i["paper_red_len_word"] / section_words) * 100, # str(round(repeat_rate, 1)) + "%"
|
|
# round(repetition_rate, 3) * 100
|
|
"thesis_title": i["title"],
|
|
"thesis_link": "",
|
|
"thesis_publish": i["degree"],
|
|
"thesis_repeat_word": i["paper_red_len_word"],
|
|
"thesis_teacher": "",
|
|
"paper_len_word": i["paper_len_word"]
|
|
}
|
|
else:
|
|
chongfuwendang[thesis_info]["thesis_repeat_word"] += i["paper_red_len_word"]
|
|
chongfuwendang[thesis_info]["thesis_repeat_rate"] = (chongfuwendang[thesis_info]["thesis_repeat_word"] /
|
|
section_words) * 100
|
|
chongfuwendang = sorted(chongfuwendang.items(),
|
|
key=lambda x: x[1]["thesis_repeat_rate"], reverse=False)
|
|
|
|
|
|
chongfuwendang_list = []
|
|
|
|
for i in chongfuwendang:
|
|
chongfuwendang_dan = i[1]
|
|
print(chongfuwendang_dan)
|
|
chongfuwendang_dan["thesis_repeat_rate"] = str(round(chongfuwendang_dan["thesis_repeat_rate"], 1)) + "%"
|
|
chongfuwendang_list.append(chongfuwendang_dan)
|
|
|
|
|
|
return chongfuwendang_list
|
|
|
|
|
|
def total_data_func(section_data_list):
|
|
'''
|
|
总体数据
|
|
:return:
|
|
'''
|
|
# "end_page_index": 0,
|
|
# "name": "第1部分",
|
|
# "repeat_rate": repeat_rate,
|
|
# "repeat_words": repeat_words,
|
|
# "start_page_index": 0,
|
|
# "words": section_words,
|
|
# "original_text": original_text,
|
|
# "original_text_oneself": original_text,
|
|
# "original_text_contrast/重复的对比详细信息": original_text_contrast,
|
|
# "repeat_quote_info/重复的引用信息": repeat_quote_info
|
|
|
|
repeat_words = 0
|
|
words = 0
|
|
|
|
for i in section_data_list:
|
|
repeat_words += i["repeat_words"]
|
|
words += i["words"]
|
|
|
|
baifenbi = (repeat_words / words) *100
|
|
exclude_personal_rate = str(round(baifenbi, 1)) + "%"
|
|
exclude_quote_rate = str(round(baifenbi, 1)) + "%"
|
|
single_max_rate = section_data_list[0]["repeat_quote_info"][0]["thesis_repeat_rate"]
|
|
single_max_repeat_words = section_data_list[0]["repeat_quote_info"][0]["thesis_repeat_word"]
|
|
total_repeat_rate = str(round(baifenbi, 1)) + "%"
|
|
total_repeat_words = repeat_words
|
|
total_words = words
|
|
|
|
print(exclude_personal_rate)
|
|
|
|
return {
|
|
"back_repeat_words": "",
|
|
"exclude_personal_rate": exclude_personal_rate,
|
|
"exclude_quote_rate": exclude_quote_rate,
|
|
"front_repeat_words": "",
|
|
"single_max_rate": single_max_rate,
|
|
"single_max_repeat_words": single_max_repeat_words,
|
|
"suspected_paragraph": "",
|
|
"suspected_paragraph_max_repeat_words": "",
|
|
"suspected_paragraph_min_repeat_words": "",
|
|
"total_paragraph": "",
|
|
"total_repeat_rate": total_repeat_rate,
|
|
"total_repeat_words": total_repeat_words,
|
|
"total_words": total_words,
|
|
"tables": 0
|
|
}
|
|
|
|
|
|
def section_data_func_dan():
|
|
'''
|
|
章节信息单个
|
|
:return:
|
|
'''
|
|
# {
|
|
# "section_name": "章节名称",
|
|
# "section_repeat_rate": "重复率",
|
|
# "section_repeat_words": "重复字数",
|
|
# "section_words": "章节字数",
|
|
# "oneself_repeat_words": "去除本人后重复字数",
|
|
# "reference_repeat_words": "去除引用后重复字数",
|
|
# "section_oneself_rate": "去除本人后重复率"
|
|
# }
|
|
|
|
return {
|
|
"section_name": "",
|
|
"section_repeat_rate": "",
|
|
"section_repeat_words": "",
|
|
"section_words": "",
|
|
"oneself_repeat_words": "",
|
|
"reference_repeat_words": "",
|
|
"section_oneself_rate": ""
|
|
}
|
|
|
|
|
|
def section_data_func(section_details):
|
|
'''
|
|
章节信息
|
|
:return:
|
|
'''
|
|
# "end_page_index": 0,
|
|
# "name": "第1部分",
|
|
# "repeat_rate": repeat_rate,
|
|
# "repeat_words": repeat_words,
|
|
# "start_page_index": 0,
|
|
# "words": section_words,
|
|
# "original_text": original_text,
|
|
# "original_text_oneself": original_text,
|
|
# "original_text_contrast/重复的对比详细信息": original_text_contrast,
|
|
# "repeat_quote_info/重复的引用信息": repeat_quote_info
|
|
|
|
section_name = section_details["name"]
|
|
section_repeat_rate = section_details["repeat_rate"]
|
|
section_repeat_words = section_details["repeat_words"]
|
|
section_words = section_details["words"]
|
|
oneself_repeat_words = section_details["repeat_words"]
|
|
reference_repeat_words = section_details["repeat_words"]
|
|
section_oneself_rate = section_details["repeat_rate"]
|
|
|
|
return {
|
|
"section_name": section_name,
|
|
"section_repeat_rate": section_repeat_rate,
|
|
"section_repeat_words": section_repeat_words,
|
|
"section_words": section_words,
|
|
"oneself_repeat_words": oneself_repeat_words,
|
|
"reference_repeat_words": reference_repeat_words,
|
|
"section_oneself_rate": section_oneself_rate
|
|
}
|
|
|
|
|
|
def section_details_func(data_section_dan, paper_dict, num_words, content_list, index_content_list_dan):
|
|
'''
|
|
章节详细信息
|
|
:param original_text_contrast:
|
|
:param repeat_quote_info:
|
|
:return:
|
|
'''
|
|
original_text_contrast = []
|
|
section_repeat_rate = ""
|
|
repeat_words = 0
|
|
section_words = num_words
|
|
oneself_repeat_words = ""
|
|
reference_repeat_words = ""
|
|
section_oneself_rate = ""
|
|
original_text_list = []
|
|
|
|
for sentence_dan in data_section_dan:
|
|
original_text_contrast_dan = original_text_contrast_func(sentence_dan, paper_dict, content_list)
|
|
original_text_contrast.append(original_text_contrast_dan)
|
|
repeat_words += original_text_contrast_dan["dan_sentence_word_nums"]
|
|
original_text_list.append(original_text_contrast_dan["original_text"])
|
|
|
|
original_text = "".join(original_text_list)
|
|
repeat_rate = (repeat_words / section_words) * 100
|
|
repeat_rate = str(round(repeat_rate, 1)) + "%"
|
|
|
|
repeat_quote_info = repeat_quote_info_func(original_text_contrast, section_words)
|
|
|
|
return {
|
|
"end_page_index": 0,
|
|
"name": "第{}部分".format(str(index_content_list_dan)),
|
|
"repeat_rate": repeat_rate,
|
|
"repeat_words": repeat_words,
|
|
"start_page_index": 0,
|
|
"words": section_words,
|
|
"original_text": original_text,
|
|
"original_text_oneself": original_text,
|
|
"original_text_contrast": original_text_contrast,
|
|
"repeat_quote_info": repeat_quote_info
|
|
}
|
|
|
|
|
|
def check_dict(similar_content_control_zong, paper_dict_zong, num_words_zong, chapter_data, index_content_list):
|
|
# similar_content_control, paper_dict, num_words, title, author, content_list
|
|
'''
|
|
生成返回字典
|
|
:param similar_content_control:
|
|
:param paper_dict:
|
|
:param num_words:
|
|
:param title:
|
|
:param author:
|
|
:return:
|
|
'''
|
|
if paper_dict_zong != []:
|
|
|
|
# 模拟多个章节
|
|
section_details_list = []
|
|
for data_section_dan, paper_dict, num_words, content_list, index_content_list_dan in zip(similar_content_control_zong, paper_dict_zong, num_words_zong, chapter_data, index_content_list):
|
|
|
|
# 章节详细信息
|
|
section_details = section_details_func(data_section_dan, paper_dict, num_words, content_list, index_content_list_dan)
|
|
section_details_list.append(section_details)
|
|
|
|
# 模拟多个章节
|
|
|
|
section_data_list = []
|
|
for section_details in section_details_list:
|
|
section_data = section_data_func(section_details)
|
|
section_data_list.append(section_data)
|
|
|
|
total_data = total_data_func(section_details_list)
|
|
|
|
format = '%Y-%m-%d %H:%M:%S'
|
|
value = time.localtime(int(time.time()))
|
|
dt = time.strftime(format, value)
|
|
|
|
paper_data = {
|
|
"author": "",
|
|
"check_time": dt,
|
|
"time_range": "1900-01-01至2023-08-08",
|
|
"title": "",
|
|
"total_data": total_data,
|
|
"section_data": section_data_list,
|
|
"section_details": section_details_list
|
|
}
|
|
else:
|
|
total_data = {
|
|
"back_repeat_words": "",
|
|
"exclude_personal_rate": 0,
|
|
"exclude_quote_rate": 0,
|
|
"front_repeat_words": "",
|
|
"single_max_rate": 0,
|
|
"single_max_repeat_words": 0,
|
|
"suspected_paragraph": "",
|
|
"suspected_paragraph_max_repeat_words": "",
|
|
"suspected_paragraph_min_repeat_words": "",
|
|
"total_paragraph": "",
|
|
"total_repeat_rate": 0,
|
|
"total_repeat_words": 0,
|
|
"total_words": 0,
|
|
"tables": 0
|
|
}
|
|
|
|
section_data_list = [{
|
|
"section_name": "第一部分",
|
|
"section_repeat_rate": 0,
|
|
"section_repeat_words": 0,
|
|
"section_words": 0,
|
|
"oneself_repeat_words": 0,
|
|
"reference_repeat_words": 0,
|
|
"section_oneself_rate": 0
|
|
}]
|
|
|
|
|
|
section_details_list = [
|
|
{
|
|
"end_page_index": 0,
|
|
"name": "第1部分",
|
|
"repeat_rate": 0,
|
|
"repeat_words": 0,
|
|
"start_page_index": 0,
|
|
"words": 0,
|
|
"original_text": "",
|
|
"original_text_oneself": "",
|
|
"original_text_contrast": [],
|
|
"repeat_quote_info": []
|
|
}
|
|
]
|
|
format = '%Y-%m-%d %H:%M:%S'
|
|
value = time.localtime(int(time.time()))
|
|
dt = time.strftime(format, value)
|
|
|
|
paper_data = {
|
|
"author": "",
|
|
"check_time": dt,
|
|
"time_range": "1900-01-01至2023-08-08",
|
|
"title": "",
|
|
"total_data": total_data,
|
|
"section_data": section_data_list,
|
|
"section_details": section_details_list
|
|
}
|
|
return paper_data
|
|
|
|
|
|
def split_chapter(content_list):
|
|
'''
|
|
|
|
:param content_list:
|
|
:return: [[[sentence, sentence, ... sentence], 2000], [[sentence, sentence, ... sentence], 2000]]
|
|
'''
|
|
|
|
content_list_new = []
|
|
zishu = 9000
|
|
dangqianzishu = 0
|
|
|
|
i = 0
|
|
content_list_dan = []
|
|
while True:
|
|
if i >= len(content_list):
|
|
if content_list_dan != []:
|
|
content_list_new.append([content_list_dan, dangqianzishu])
|
|
break
|
|
content_list_dan.append(content_list[i])
|
|
dangqianzishu += len(content_list[i])
|
|
if dangqianzishu > zishu:
|
|
content_list_new.append([content_list_dan, dangqianzishu])
|
|
dangqianzishu = 0
|
|
content_list_dan = []
|
|
i += 1
|
|
|
|
return content_list_new
|
|
|
|
|
|
# def biahong_rule(s1, s2):
|
|
# '''
|
|
#
|
|
# :param s1:
|
|
# :param s2:
|
|
# :return:
|
|
# {
|
|
# "probabilities": null,
|
|
# "resilt": [
|
|
# [
|
|
# "而且受大型火灾事件的影响，会对公众心理造成相应的伤害，火灾发生后让人们产生恐惧、烦躁等心理现象，这也会妨碍对火灾的管理。1大型商业建筑火灾的特点大型企业的商业建筑一般设计为商业综合体，其中包括办公，餐饮，商铺，娱乐等活动场所，具有建筑市场规模大、人数多、火灾荷载大、消防救援难等特点，其建筑火灾具有以下特点。1.1可燃物种类多，火灾荷载集中大型企业综合商业建筑设计一般主要包括室内步行街和大量的商铺柜台等部分。",
|
|
# "火灾的管理。1大型商业建筑火灾的特点大型企业的商业建筑一般设计为商业综合体，其中包括办公，餐饮，商铺，娱乐等活动场所，具有建筑市场规模大、人数多、火灾荷载大、消防救援难等特点，其建筑火灾具有以下特点。1.1可燃物种类多，火灾荷载集中大型企业综合商业建筑设计一般主要包括室内步行街和大量的商铺柜台等部分",
|
|
# "1大型商业建筑火灾的特点大型商业建筑一般为商业综合体,其中包含商铺、娱乐等多种场所,具有建筑规模大、人员多、火灾荷载大、扑救困难等特点,其建筑火灾有以下特点。1.1可燃物种类多,火灾荷载集中大型综合体商业建筑中一般包含室内步行街与大量百货专柜等部分。1111",
|
|
# "1大型商业建筑火灾的特点大型商业建筑一般为商业综合体,其中包含商铺、娱乐等多种场所,具有建筑规模大、人员多、火灾荷载大、扑救困难等特点,其建筑火灾有以下特点。1.1可燃物种类多,火灾荷载集中大型综合体商业建筑中一般包含室内步行街与大量百货专柜等部分。1111",
|
|
# [
|
|
# 54,
|
|
# 204,
|
|
# 0,
|
|
# 126
|
|
# ]
|
|
# ],
|
|
# ]
|
|
# }
|
|
# '''
|
|
#
|
|
# id_start_1_dan_best = 0
|
|
# id_end_1_dan_best = len(s1) -1
|
|
# id_start_2_dan_best = 0
|
|
# id_end_2_dan_best = len(s2) -1
|
|
# sim_score_best_best = 0
|
|
#
|
|
# while True:
|
|
# if sim_score_best >= 0.75:
|
|
# break
|
|
# else:
|
|
# id_start_1_dan = 0
|
|
# id_end_1_dan = len(s1) - 1
|
|
# id_start_2_dan = 0
|
|
# id_end_2_dan = len(s2) - 1
|
|
#
|
|
# sen_list = [
|
|
# [s1[id_start_1_dan_best+1:id_end_1_dan_best +1], s2[id_start_2_dan_best:id_end_2_dan_best +1]],
|
|
# [s1[id_start_1_dan_best:id_end_1_dan_best], s2[id_start_2_dan_best:id_end_2_dan_best +1]],
|
|
# [s1[id_start_1_dan_best :id_end_1_dan_best + 1], s2[id_start_2_dan_best + 1:id_end_2_dan_best + 1]],
|
|
# [s1[id_start_1_dan_best:id_end_1_dan_best + 1], s2[id_start_2_dan_best:id_end_2_dan_best]]
|
|
# ]
|
|
# for i in sen_list:
|
|
# sim_score = jaccard_similarity(i[0], i[1])
|
|
#
|
|
#
|
|
# return
|
|
|
|
def chapter_check(dan_chapter_data, recall_data_list):
|
|
# =============================================================================================
|
|
# 多进程算法
|
|
# rouge算法查重
|
|
# t1_0 = time.time()
|
|
# rst = []
|
|
# p = Pool(nums_cpus) # 进程池中含有n个子进程
|
|
#
|
|
# print("num_words", num_words)
|
|
# for i in range(len(content_list)):
|
|
# text = content_list[i]
|
|
# a = p.apply_async(rouge_pre_m, args=(text, recall_data_list,))
|
|
# rst.append(a)
|
|
# p.close()
|
|
# p.join() # 等待所有子进程执行完毕。调用join()之前必须先调用close()，调用close()之后就不能继续添加新的Process了。
|
|
#
|
|
# print("筛选句子完成")
|
|
# rst = [i.get() for i in rst]
|
|
#
|
|
# t2_0 = time.time()
|
|
# print(t2_0- t1_0)
|
|
|
|
# =========================================================================================================
|
|
|
|
rst = []
|
|
for i in range(len(dan_chapter_data)):
|
|
text = dan_chapter_data[i]
|
|
rst.append(rouge_pre_m(text, recall_data_list))
|
|
|
|
# ========================================================================================================
|
|
|
|
data_zong = []
|
|
for i in range(len(rst)):
|
|
# print(rst[i])
|
|
data_zong.append(rst[i])
|
|
|
|
t0 = time.time()
|
|
# bert算法查重
|
|
# for text in content_list:
|
|
# bert_pre_list = bert_check(text, recall_data_list)
|
|
# data_zong.append(bert_pre_list)
|
|
t1 = time.time()
|
|
original_dict = []
|
|
|
|
# 找出相似的句子序号
|
|
bool_check_sentense = [] # [[[1, 223],[1, 226], [1, 562]],[[2, 243],[2, 226], [2, 561]]]
|
|
|
|
|
|
# rouge算法
|
|
for i in range(len(data_zong)):
|
|
bool_check_sentense_dan = [] # [[1, 223],[1, 226], [1, 562]]
|
|
for j in range(len(data_zong[i])):
|
|
if data_zong[i][j][1] > 0.3:
|
|
# print("data_zong[{}][{}]".format(i,j), data_zong[i][j][0])
|
|
bool_check_sentense_dan.append([i, data_zong[i][j][0]])
|
|
if bool_check_sentense_dan != []:
|
|
bool_check_sentense.append(bool_check_sentense_dan)
|
|
|
|
# 继续用rouge方法筛选
|
|
|
|
if bool_check_sentense == []:
|
|
pass
|
|
bool_check_sentense = rouge_pre_m_1(bool_check_sentense, dan_chapter_data,
|
|
recall_data_list) # [[[1, 223],[1, 226], [1, 562]],[[2, 243],[2, 226], [2, 561]]]
|
|
|
|
print("bool_check_sentense", bool_check_sentense)
|
|
print("找出相似的句子序号完成")
|
|
|
|
# print("data_zong", data_zong)
|
|
biao_red = biaohong(bool_check_sentense, data_zong,
|
|
recall_data_list) # [[[[0, 1, 2], [479, 480, 481]],[[0, 1, 2], [471, 472, 473]]], [[[3, 4, 5], [481, 482, 483]], [[3, 4, 5], [461, 462, 463]]]]
|
|
|
|
print("biao_red", str(biao_red))
|
|
|
|
original_sentence_index = []
|
|
# for i in biao_red:
|
|
# for j in i:
|
|
# original_sentence_index.append(j[0])
|
|
|
|
sentence_0_list = []
|
|
sentence_1_list = []
|
|
sim_paper_name = []
|
|
|
|
for i in range(len(biao_red)):
|
|
for j in range(len(biao_red[i])):
|
|
print("i,j",i, j)
|
|
# if recall_data_list[biao_red[i][j][1][0]][1] == recall_data_list[biao_red[i][j][1][1]][1] == recall_data_list[biao_red[i][j][1][2]][1]:
|
|
# sentence_0_list.append("".join([content_list[biao_red[i][j][0][0]], content_list[biao_red[i][j][0][1]], content_list[biao_red[i][j][0][2]]]))
|
|
# sentence_1_list.append(
|
|
# "".join([recall_data_list[biao_red[i][j][1][0]][0], recall_data_list[biao_red[i][j][1][1]][0], recall_data_list[biao_red[i][j][1][2]][0]]))
|
|
# sim_paper_name.append(recall_data_list[biao_red[i][j][1][0]][1])
|
|
# else:
|
|
# continue
|
|
|
|
file_name = recall_data_list[biao_red[i][j][1][1]][1]
|
|
sentence_0_list_dan = []
|
|
sentence_1_list_dan = []
|
|
sentence_0_list_dan_index = []
|
|
# houxuna_file_list = [
|
|
# [recall_data_list[biao_red[i][j][1][0]][1], dan_chapter_data[biao_red[i][j][0][0]],
|
|
# recall_data_list[biao_red[i][j][1][0]][0]],
|
|
# [recall_data_list[biao_red[i][j][1][1]][1], dan_chapter_data[biao_red[i][j][0][1]],
|
|
# recall_data_list[biao_red[i][j][1][1]][0]],
|
|
# [recall_data_list[biao_red[i][j][1][2]][1], dan_chapter_data[biao_red[i][j][0][2]],
|
|
# recall_data_list[biao_red[i][j][1][2]][0]]
|
|
# ]
|
|
|
|
sentence_0_list_dan = [dan_chapter_data[biao_red[i][j][0][index_simsentence]] for index_simsentence in range(len(biao_red[i][j][0]))]
|
|
houxuna_file_list = [[recall_data_list[biao_red[i][j][1][index_simsentence]][1], recall_data_list[biao_red[i][j][1][index_simsentence]][0]] for index_simsentence in range(len(biao_red[i][j][0]))]
|
|
|
|
for dan_sen_info in houxuna_file_list:
|
|
if dan_sen_info[0] == file_name:
|
|
sentence_1_list_dan.append(dan_sen_info[1])
|
|
if sentence_0_list_dan != [] and sentence_1_list_dan != []:
|
|
sentence_0_list.append("".join(sentence_0_list_dan))
|
|
sentence_1_list.append("".join(sentence_1_list_dan))
|
|
original_sentence_index.append(biao_red[i][j][0])
|
|
sim_paper_name.append(recall_data_list[biao_red[i][j][1][1]][1])
|
|
|
|
print("待标红句子筛选完成")
|
|
sentence_0_list_new = []
|
|
sentence_1_list_new = []
|
|
|
|
for i in zip(sentence_0_list, sentence_1_list):
|
|
if len(i[0]) + len(i[1]) < 1200:
|
|
sentence_0_list_new.append(i[0])
|
|
sentence_1_list_new.append(i[1])
|
|
else:
|
|
print(len(i[0]) + len(i[1]))
|
|
continue
|
|
t2 = time.time()
|
|
|
|
print()
|
|
for i in sentence_0_list_new:
|
|
print("sentence_0_list_new", i)
|
|
if sentence_0_list_new == sentence_1_list_new == []:
|
|
paper_dict = []
|
|
else:
|
|
print("sentence_0_list_new", len(sentence_0_list_new))
|
|
print("sentence_1_list_new", len(sentence_1_list_new))
|
|
|
|
# ================================================================================================
|
|
# 深度学习标红
|
|
paper_dict = biaohong_bert_predict(sentence_0_list_new, sentence_1_list_new)
|
|
|
|
# 策略标红
|
|
|
|
|
|
|
|
t3 = time.time()
|
|
print("标红完成")
|
|
print("标红时间", t3 - t2)
|
|
|
|
print("paper_dict", paper_dict)
|
|
print("sentence_0_list_new", sentence_0_list_new)
|
|
print("sentence_1_list_new", sentence_1_list_new)
|
|
print("sim_paper_name", sim_paper_name)
|
|
similar_content_control = [[]]
|
|
# with open("data/10235513_大型商业建筑人员疏散设计研究_沈福禹/paper_dict.json", "w") as f:
|
|
# json.dump(paper_dict, f, ensure_ascii=False)
|
|
if sentence_0_list_new != []:
|
|
sentence_0_list_new_cursor = sentence_0_list_new[0]
|
|
for paper_dict_dan_id, sentence_0_dan, sentence_1_dan, sim_paper_name_dan, original_sentence_index_dan in zip(
|
|
range(len(paper_dict)),
|
|
sentence_0_list_new,
|
|
sentence_1_list_new,
|
|
sim_paper_name,
|
|
original_sentence_index):
|
|
|
|
if sentence_0_list_new_cursor != sentence_0_dan:
|
|
similar_content_control.append(
|
|
[[paper_dict_dan_id, sentence_0_dan, sentence_1_dan, sim_paper_name_dan, original_sentence_index_dan]])
|
|
sentence_0_list_new_cursor = sentence_0_dan
|
|
else:
|
|
similar_content_control[-1].append(
|
|
[paper_dict_dan_id, sentence_0_dan, sentence_1_dan, sim_paper_name_dan, original_sentence_index_dan])
|
|
return similar_content_control, paper_dict
|
|
|
|
|
|
def accurate_check_rouge(
|
|
text_paper,
|
|
recall_data_list
|
|
):
|
|
'''
|
|
精确查重出相似句子
|
|
:param text:
|
|
:param recall_data_list: list [[sentence, filename],[sentence, filename],[sentence, filename]]
|
|
:return:
|
|
'''
|
|
# 文本处理
|
|
# content_list = []
|
|
print("text_paper", len(text_paper))
|
|
text_paper = str(text_paper).replace("。\n", "。")
|
|
content_list_old = text_paper.split("。")
|
|
|
|
sentence_word_nums = 0
|
|
|
|
# 前处理，筛选句子
|
|
content_list = []
|
|
for i in content_list_old:
|
|
if len(i) <= 7:
|
|
continue
|
|
elif len(i) < 300:
|
|
content_list.append(i + "。")
|
|
if i == "。":
|
|
continue
|
|
# 分章
|
|
content_list_zong = split_chapter(content_list)
|
|
|
|
similar_content_control_zong = []
|
|
paper_dict_zong = []
|
|
num_words_zong = []
|
|
chapter_data = []
|
|
index_content_list = []
|
|
|
|
for index_content_list_zong in range(len(content_list_zong)):
|
|
dan_chapter_data, dan_chapter_num_words = content_list_zong[index_content_list_zong][0], content_list_zong[index_content_list_zong][1]
|
|
|
|
similar_content_control, paper_dict = chapter_check(dan_chapter_data, recall_data_list)
|
|
similar_content_control_zong.append(similar_content_control)
|
|
paper_dict_zong.append(paper_dict)
|
|
num_words_zong.append(dan_chapter_num_words)
|
|
chapter_data.append(dan_chapter_data)
|
|
index_content_list.append(index_content_list_zong)
|
|
|
|
paper_data = check_dict(similar_content_control_zong, paper_dict_zong, num_words_zong, chapter_data, index_content_list)
|
|
# data = [similar_content_control]
|
|
#
|
|
# # 模拟多个章节
|
|
# section_details_list = []
|
|
# for data_dan in data:
|
|
# data_section_dan = data_dan
|
|
#
|
|
# # 章节详细信息
|
|
# section_details = section_details_func(data_section_dan, paper_dict, num_words)
|
|
# section_details_list.append(section_details)
|
|
#
|
|
# # 模拟多个章节
|
|
#
|
|
# section_data_list = []
|
|
# for section_details in section_details_list:
|
|
# section_data = section_data_func(section_details)
|
|
# section_data_list.append(section_data)
|
|
#
|
|
# total_data = total_data_func(section_details_list)
|
|
#
|
|
# format = '%Y-%m-%d %H:%M:%S'
|
|
# value = time.localtime(int(time.time()))
|
|
# dt = time.strftime(format, value)
|
|
#
|
|
# paper_data = {
|
|
# "author": author,
|
|
# "check_time": dt,
|
|
# "time_range": "1900-01-01至2023-08-08",
|
|
# "title": title,
|
|
# "total_data": total_data,
|
|
# "section_data": section_data_list,
|
|
# "section_details": section_details_list
|
|
# }
|
|
return paper_data
|
|
|
|
|
|
def biaohong(bool_check_sentense, data_zong, df_train_nuoche):
|
|
'''
|
|
标红的序号 [[0,1,2],[3,4,5]]
|
|
:param bool_check_sentense: # [[[1, 223],[1, 226], [1, 562]],[[2, 243],[2, 226], [2, 561]]]
|
|
:return: list # [[[[0, 1, 2], [479, 480, 481]],[[0, 1, 2], [471, 472, 473]]], [[[3, 4, 5], [481, 482, 483]], [[3, 4, 5], [461, 462, 463]]]]
|
|
'''
|
|
|
|
# print("bool_check_sentense", bool_check_sentense)
|
|
biao_red = []
|
|
i = 0
|
|
start = -1
|
|
end = -1
|
|
tiaochu = False
|
|
while True:
|
|
# if i >= len(bool_check_sentense) or bool_check_sentense[i][0] + 1 >= len(data_zong) or bool_check_sentense[i][1] \
|
|
# + 1 >= len(df_train_nuoche):
|
|
# break
|
|
|
|
if i >= len(bool_check_sentense):
|
|
break
|
|
|
|
for j in bool_check_sentense[i]:
|
|
# print("j", j)
|
|
if j[0] + 1 > len(data_zong):
|
|
tiaochu = True
|
|
break
|
|
|
|
# if bool_check_sentense[i][0][0] + 1 >= len(data_zong):
|
|
# if bool_check_sentense[]
|
|
# bool_check_sentense[i][0][0] + 1 = bool_check_sentense[i + 1][0][0]
|
|
# break
|
|
|
|
for j in bool_check_sentense[i]:
|
|
if j[1] + 1 >= len(df_train_nuoche):
|
|
tiaochu = True
|
|
break
|
|
|
|
if tiaochu == True:
|
|
break
|
|
|
|
# elif bool_check_sentense[i-1][0][0] == start:
|
|
# biao_red_dan = []
|
|
# for j in range(len(bool_check_sentense[i-1])): # bool_check_sentense: [[[1, 223],[1, 226], [1, 562]],[[2, 243],[2, 226], [2, 561]]]
|
|
# biao_red_dan.append([[bool_check_sentense[i-1][j][0], bool_check_sentense[i-1][j][0]+ 1, bool_check_sentense[i-1][j][0] + 2],
|
|
# [bool_check_sentense[i-1][j][1] - 1, bool_check_sentense[i-1][j][1], bool_check_sentense[i+1][j][1] + 1]])
|
|
# biao_red.append(biao_red_dan)
|
|
#
|
|
# elif bool_check_sentense[i+1][0][0] == end:
|
|
# biao_red_dan = []
|
|
# for j in range(len(bool_check_sentense[i+1])): # bool_check_sentense: [[[1, 223],[1, 226], [1, 562]],[[2, 243],[2, 226], [2, 561]]]
|
|
# biao_red_dan.append([[bool_check_sentense[i+1][j][0]-2, bool_check_sentense[i+1][j][0]-1, bool_check_sentense[i+1][j][0]],
|
|
# [bool_check_sentense[i+1][j][1] - 1, bool_check_sentense[i+1][j][1], bool_check_sentense[i+1][j][1] + 1]])
|
|
# biao_red.append(biao_red_dan)
|
|
|
|
elif i == len(bool_check_sentense)-1:
|
|
if end == bool_check_sentense[i][0][0]:
|
|
i += 1
|
|
break
|
|
elif bool_check_sentense[i][0][0]-1 == end + 1 and bool_check_sentense[i][0][0] == len(data_zong) -1:
|
|
index_list = [ii for ii in range(bool_check_sentense[i][0][0]-1, bool_check_sentense[i][0][0] + 1)]
|
|
elif bool_check_sentense[i][0][0]-1 == end and bool_check_sentense[i][0][0] == len(data_zong) -1:
|
|
index_list = [ii for ii in range(bool_check_sentense[i][0][0], bool_check_sentense[i][0][0] + 1)]
|
|
elif bool_check_sentense[i][0][0]-1 > end + 1 and bool_check_sentense[i][0][0] == len(data_zong) -1:
|
|
index_list = [ii for ii in range(bool_check_sentense[i][0][0] - 1, bool_check_sentense[i][0][0] + 1)]
|
|
else:
|
|
index_list = [ii for ii in range(bool_check_sentense[i][0][0] - 1, bool_check_sentense[i][0][0] + 2)]
|
|
|
|
biaohongset = set()
|
|
biao_red_dan = []
|
|
for j in range(len(bool_check_sentense[
|
|
i])): # bool_check_sentense: [[[1, 223],[1, 226], [1, 562]],[[2, 243],[2, 226], [2, 561]]]
|
|
if bool_check_sentense[i][j][1] not in biaohongset:
|
|
biao_red_dan.append([index_list,
|
|
[bool_check_sentense[i][j][1] - 1, bool_check_sentense[i][j][1],
|
|
bool_check_sentense[i][j][1] + 1]])
|
|
biaohongset.add(bool_check_sentense[i][j][1] - 1)
|
|
biaohongset.add(bool_check_sentense[i][j][1])
|
|
biaohongset.add(bool_check_sentense[i][j][1] + 1)
|
|
else:
|
|
continue
|
|
|
|
i += 1
|
|
biao_red.append(biao_red_dan)
|
|
break
|
|
|
|
elif bool_check_sentense[i][0][0] - 1 == start:
|
|
i += 1
|
|
continue
|
|
elif bool_check_sentense[i][0][0] == end:
|
|
i += 1
|
|
continue
|
|
elif bool_check_sentense[i][0][0] - 1 == end:
|
|
i += 1
|
|
continue
|
|
else:
|
|
biaohongset = set()
|
|
biao_red_dan = []
|
|
for j in range(len(bool_check_sentense[i])): # bool_check_sentense: [[[1, 223],[1, 226], [1, 562]],[[2, 243],[2, 226], [2, 561]]]
|
|
if bool_check_sentense[i][j][1] not in biaohongset:
|
|
biao_red_dan.append([[bool_check_sentense[i][j][0] - 1, bool_check_sentense[i][j][0], bool_check_sentense[i][j][0] + 1],
|
|
[bool_check_sentense[i][j][1] - 1, bool_check_sentense[i][j][1], bool_check_sentense[i][j][1] + 1]])
|
|
biaohongset.add(bool_check_sentense[i][j][1] - 1)
|
|
biaohongset.add(bool_check_sentense[i][j][1])
|
|
biaohongset.add(bool_check_sentense[i][j][1] + 1)
|
|
else:
|
|
continue
|
|
|
|
start = bool_check_sentense[i][0][0] - 1
|
|
end = bool_check_sentense[i][0][0] + 1
|
|
|
|
if bool_check_sentense[i-1][0][0] == start:
|
|
for j in range(len(bool_check_sentense[i-1])): # bool_check_sentense: [[[1, 223],[1, 226], [1, 562]],[[2, 243],[2, 226], [2, 561]]]
|
|
if bool_check_sentense[i - 1][j][1] not in biaohongset:
|
|
biao_red_dan.append([[bool_check_sentense[i-1][j][0], bool_check_sentense[i-1][j][0] + 1, bool_check_sentense[i-1][j][0] + 2],
|
|
[bool_check_sentense[i-1][j][1] - 1, bool_check_sentense[i-1][j][1], bool_check_sentense[i-1][j][1] + 1]])
|
|
biaohongset.add(bool_check_sentense[i-1][j][1] - 1)
|
|
biaohongset.add(bool_check_sentense[i-1][j][1])
|
|
biaohongset.add(bool_check_sentense[i-1][j][1] + 1)
|
|
else:
|
|
continue
|
|
|
|
if bool_check_sentense[i+1][0][0] == end:
|
|
for j in range(len(bool_check_sentense[i+1])): # bool_check_sentense: [[[1, 223],[1, 226], [1, 562]],[[2, 243],[2, 226], [2, 561]]]
|
|
if bool_check_sentense[i + 1][j][1] not in biaohongset:
|
|
biao_red_dan.append([[bool_check_sentense[i+1][j][0]-2, bool_check_sentense[i+1][j][0]-1, bool_check_sentense[i+1][j][0]],
|
|
[bool_check_sentense[i+1][j][1] - 1, bool_check_sentense[i+1][j][1], bool_check_sentense[i+1][j][1] + 1]])
|
|
biaohongset.add(bool_check_sentense[i+1][j][1] - 1)
|
|
biaohongset.add(bool_check_sentense[i+1][j][1])
|
|
biaohongset.add(bool_check_sentense[i+1][j][1] + 1)
|
|
else:
|
|
continue
|
|
|
|
i += 1
|
|
biao_red.append(biao_red_dan)
|
|
|
|
return biao_red # [[[[0, 1, 2], [479, 480, 481]],[[0, 1, 2], [471, 472, 473]]], [[[3, 4, 5], [481, 482, 483]], [[3, 4, 5], [461, 462, 463]]]]
|
|
|
|
|
|
def dialog_line_parse(url, text):
|
|
"""
|
|
将数据输入模型进行分析并输出结果
|
|
:param url: 模型url
|
|
:param text: 进入模型的数据
|
|
:return: 模型返回结果
|
|
"""
|
|
|
|
response = requests.post(
|
|
url,
|
|
json=text,
|
|
timeout=100000
|
|
)
|
|
if response.status_code == 200:
|
|
return response.json()
|
|
else:
|
|
# logger.error(
|
|
# "【{}】 Failed to get a proper response from remote "
|
|
# "server. Status Code: {}. Response: {}"
|
|
# "".format(url, response.status_code, response.text)
|
|
# )
|
|
print("【{}】 Failed to get a proper response from remote "
|
|
"server. Status Code: {}. Response: {}"
|
|
"".format(url, response.status_code, response.text))
|
|
print(text)
|
|
return {}
|
|
|
|
|
|
def is_english_char(char):
|
|
code = ord(char)
|
|
return 32 <= code <= 126
|
|
|
|
|
|
def original_text_marked_red(text_original, bert_text, start, end):
|
|
'''
|
|
把原文标红字段找到
|
|
:param text_original:
|
|
:param bert_text:
|
|
:param bert_text_pre:
|
|
:return:
|
|
'''
|
|
try:
|
|
fuhao = ["\n"]
|
|
up_pointer = 0
|
|
down_pointer = 0
|
|
|
|
pointer_list = []
|
|
|
|
bert_text_list = list(bert_text)
|
|
bert_text_list.insert(start, "<red>")
|
|
bert_text_list.insert(end + 2, "</red>")
|
|
|
|
text_original_list = list(text_original)
|
|
|
|
up = 0
|
|
down = 0
|
|
|
|
while True:
|
|
if up == len(text_original_list):
|
|
break
|
|
|
|
if text_original_list[up] == bert_text_list[down]:
|
|
up += 1
|
|
down += 1
|
|
|
|
else:
|
|
if bert_text_list[down] == "<red>":
|
|
down += 1
|
|
elif bert_text_list[down] == "</red>":
|
|
down += 1
|
|
else:
|
|
bert_text_list.insert(down, text_original_list[up])
|
|
up += 1
|
|
down += 1
|
|
|
|
bert_text = "".join(bert_text_list)
|
|
return True, bert_text
|
|
except:
|
|
print("句子标红报错")
|
|
print(text_original, bert_text)
|
|
return False, ""
|
|
|
|
|
|
def biaohong_bert_predict(sentence_0_list, sentence_1_list):
|
|
'''
|
|
找出标红字符
|
|
:param bool_check_sentense:
|
|
:return:
|
|
'''
|
|
|
|
paper_dict = \
|
|
dialog_line_parse("http://192.168.31.74:16003/",
|
|
{"sentence_0": sentence_0_list, "sentence_1": sentence_1_list})[
|
|
"resilt"]
|
|
|
|
return paper_dict
|
|
|
|
|
|
def ulit_text(title, text):
|
|
data = []
|
|
try:
|
|
text = json.loads(text)["content"]
|
|
except:
|
|
pass
|
|
|
|
text = text.strip().replace("\n", "").replace(" ", "").replace("。", "。\n")
|
|
text_list = text.split("\n")
|
|
|
|
for i in text_list:
|
|
data.append([i, title])
|
|
return data
|
|
|
|
|
|
def run_query(conn, sql, params):
|
|
with conn.cursor() as cursor:
|
|
cursor.execute(sql, params)
|
|
result = cursor.fetchall()
|
|
return result
|
|
|
|
|
|
# def processing_one_text(paper_id):
|
|
# conn = pymysql.connect(
|
|
# host='192.168.31.145',
|
|
# port=3306,
|
|
# user='root',
|
|
# password='123456',
|
|
# db='zhiwang_db',
|
|
# charset='utf8mb4',
|
|
# cursorclass=pymysql.cursors.DictCursor
|
|
# )
|
|
#
|
|
# sql = 'SELECT * FROM main_table_paper_detail_message WHERE doc_id=%s'
|
|
# params = (paper_id,)
|
|
#
|
|
# result = run_query(conn, sql, params)
|
|
#
|
|
# conn.close()
|
|
# print(result[0]['title'], result[0]['author'])
|
|
# title = result[0]['title']
|
|
# author = result[0]['author']
|
|
# degree = result[0]['degree']
|
|
# year = result[0]['content'].split("/")[5]
|
|
# content_path = result[0]['content']
|
|
#
|
|
# try:
|
|
# with open(content_path, encoding="utf-8") as f:
|
|
# text = f.read()
|
|
# except:
|
|
# with open(content_path, encoding="gbk") as f:
|
|
# text = f.read()
|
|
#
|
|
# paper_info = {
|
|
# "title": title,
|
|
# "author": author,
|
|
# "degree": degree,
|
|
# "year": year,
|
|
# "paper_len_word": len(text)
|
|
# }
|
|
# data = ulit_text(paper_info, text)
|
|
# return data
|
|
|
|
|
|
from clickhouse_driver import Client
|
|
|
|
class PureClient:
|
|
def __init__(self, database='mini_check'):
|
|
# 只需要写本地地址
|
|
self.client = Client(host=f'{"192.168.31.74"}', port=9000, user='default',
|
|
password='zhicheng123*', database=database)
|
|
|
|
def run(self, sql):
|
|
client = self.client
|
|
collection = client.query_dataframe(sql)
|
|
return collection
|
|
|
|
def processing_one_text(user_uuid):
|
|
|
|
pureclient = PureClient()
|
|
print("paper_id", user_uuid)
|
|
sql = f"SELECT * FROM user_table WHERE user_uuid='{user_uuid}'"
|
|
result = pureclient.run(sql)
|
|
return result
|
|
|
|
|
|
def ulit_recall_paper(uuid_uesr):
|
|
'''
|
|
对返回的十篇文章路径读取并解析
|
|
:param recall_data_list_path:
|
|
:return data: list [[sentence, filename],[sentence, filename],[sentence, filename]]
|
|
'''
|
|
|
|
# data = []
|
|
# for path in recall_data_list_path:
|
|
# filename = path.split("/")[-1]
|
|
# with open(path, encoding="gbk") as f:
|
|
# text = f.read()
|
|
# text_list = text.split("\n")
|
|
# for sentence in text_list:
|
|
# if sentence != "":
|
|
# data.append([sentence, filename])
|
|
# return data
|
|
res = processing_one_text(uuid_uesr)
|
|
res_list = res.values.tolist()
|
|
|
|
data = []
|
|
for res_dan in res_list:
|
|
user_uuid = res_dan[0]
|
|
file_path = res_dan[1]
|
|
is_delete = res_dan[2]
|
|
if is_delete == 1:
|
|
try:
|
|
with open(file_path, encoding="gbk") as f:
|
|
text = f.read()
|
|
except:
|
|
with open(file_path, encoding="utf-8") as f:
|
|
text = f.read()
|
|
text_list = text.split("。")
|
|
|
|
filename = file_path.split("/")[-1].split(".")[0]
|
|
for sentence in text_list:
|
|
if sentence != "":
|
|
data.append([sentence.strip("\n"), filename])
|
|
|
|
return data
|
|
|
|
|
|
def recall_10(queue_uuid, title, abst_zh, content):
|
|
'''
|
|
宇鹏召回接口
|
|
:param paper_name:
|
|
:return:
|
|
'''
|
|
|
|
request_json = {
|
|
"uuid": queue_uuid,
|
|
"title": title,
|
|
"abst_zh": abst_zh,
|
|
"content": content
|
|
}
|
|
print(request_json)
|
|
dialog_line_parse("http://192.168.31.145:50004/check1", request_json)
|
|
|
|
|
|
|
|
def uilt_content(content):
|
|
zhaiyao_list = ["摘要"]
|
|
zhaiyao_en_list = ["Abstract", "abstract"]
|
|
mulu_list = ["目录"]
|
|
key_word_list = ["关键词"]
|
|
caikanwenxian = ["参考文献"]
|
|
key_word_bool = False
|
|
key_word_str = ""
|
|
zhaiyao_bool = False
|
|
zhaiyao_en_bool = False
|
|
zhaiyao_str = ""
|
|
zhaiyao_en_str = ""
|
|
mulu_str = ""
|
|
zhaiyao_text = ""
|
|
mulu_bool = False
|
|
|
|
pantten_zhaiyao = '(摘\s*要)'
|
|
result_biaoti_list = re.findall(pantten_zhaiyao, content)
|
|
if len(result_biaoti_list) != 0:
|
|
zhaiyao_str = result_biaoti_list[0]
|
|
zhaiyao_bool = True
|
|
else:
|
|
zhaiyao_bool = False
|
|
|
|
for i in zhaiyao_en_list:
|
|
if i in content:
|
|
zhaiyao_en_bool = True
|
|
zhaiyao_en_str = i
|
|
break
|
|
|
|
for i in mulu_list:
|
|
if i in content:
|
|
mulu_str = i
|
|
mulu_bool = True
|
|
break
|
|
|
|
for i in key_word_list:
|
|
if i in content:
|
|
key_word_str = i
|
|
key_word_bool = True
|
|
break
|
|
|
|
if zhaiyao_bool == True and key_word_bool == True:
|
|
pantten_zhaiyao = "{}(.*?){}".format(zhaiyao_str, key_word_str)
|
|
result_biaoti_list = re.findall(pantten_zhaiyao, content)
|
|
zhaiyao_text = result_biaoti_list[0]
|
|
|
|
elif zhaiyao_bool == True and zhaiyao_en_bool == True:
|
|
pantten_zhaiyao = "{}(.*?){}".format(zhaiyao_str, zhaiyao_en_str)
|
|
result_biaoti_list = re.findall(pantten_zhaiyao, content)
|
|
zhaiyao_text = result_biaoti_list[0]
|
|
|
|
elif zhaiyao_bool == True and mulu_bool == True:
|
|
pantten_zhaiyao = "{}(.*?){}".format(zhaiyao_str, mulu_str)
|
|
result_biaoti_list = re.findall(pantten_zhaiyao, content)
|
|
zhaiyao_text = result_biaoti_list[0]
|
|
|
|
if zhaiyao_text == "":
|
|
content = str(content).replace("。\n", "。")
|
|
content_list = content.split("。")
|
|
zhaiyao_text = "".join(content_list[:15])
|
|
return zhaiyao_text
|
|
|
|
|
|
def ulit_request_file(file):
|
|
file_name = file.filename
|
|
if file_name.split(".")[-1] == "txt":
|
|
file_name_save = "data/request/{}".format(file_name)
|
|
file.save(file_name_save)
|
|
try:
|
|
with open(file_name_save, encoding="gbk") as f:
|
|
content = f.read()
|
|
except:
|
|
with open(file_name_save, encoding="utf-8") as f:
|
|
content = f.read()
|
|
|
|
content = " ".join([i for i in content.split("\n") if i != ""])
|
|
abst_zh = uilt_content(content)
|
|
|
|
return abst_zh, content
|
|
|
|
|
|
# @app.route("/", methods=["POST"])
|
|
# def handle_query():
|
|
# print(request.remote_addr)
|
|
#
|
|
# # request.form.get('prompt')
|
|
# dataBases = request.form.get("dataBases")
|
|
# minSimilarity = request.form.get("minSimilarity") # txt
|
|
# minWords = request.form.get("minWords")
|
|
# title = request.form.get("title")
|
|
# author = request.form.get("author") # txt
|
|
# file = request.files.get('file')
|
|
# token = request.form.get("token")
|
|
# account = request.form.get("account")
|
|
# goodsId = request.form.get("goodsId")
|
|
# callbackUrl = request.form.get("callbackUrl")
|
|
#
|
|
#
|
|
# t0 = time.time()
|
|
# abst_zh, content = ulit_request_file(file)
|
|
#
|
|
# # 调用宇鹏查询相似十篇
|
|
# # recall_data_list_dict = recall_10(title, abst_zh, content)
|
|
#
|
|
# t1 = time.time()
|
|
# print("查找相似的50篇完成")
|
|
# with open("data/rell_json.txt") as f:
|
|
# recall_data_list_dict = eval(f.read())
|
|
#
|
|
# # 读取文章转化成格式数据
|
|
# recall_data_list = ulit_recall_paper(recall_data_list_dict)
|
|
# print("文章格式转化完成")
|
|
#
|
|
# # recall_data_list = pd.read_csv("data/10235513_大型商业建筑人员疏散设计研究_沈福禹/查重.csv", encoding="utf-8").values.tolist()
|
|
#
|
|
# # 进入精确查重系统
|
|
# print("进入精确查重系统")
|
|
# return_list = accurate_check_rouge(title, author, content, recall_data_list)
|
|
#
|
|
# print("召回50篇", t1 - t0)
|
|
#
|
|
# return_text = {"resilt": return_list, "probabilities": None, "status_code": 200}
|
|
# return jsonify(return_text) # 返回结果
|
|
|
|
|
|
# def classify_recall(): # 调用模型，设置最大batch_size
|
|
# while True:
|
|
# if redis_.llen(db_key_query) == 0: # 若队列中没有元素就继续获取
|
|
# time.sleep(3)
|
|
# continue
|
|
# query = redis_.lpop(db_key_query).decode('UTF-8') # 获取query的text
|
|
# data_dict_path = json.loads(query)
|
|
# path = data_dict_path['path']
|
|
# # text_type = data_dict["text_type"]
|
|
#
|
|
# with open(path, encoding='utf8') as f1:
|
|
# # 加载文件的对象
|
|
# data_dict = json.load(f1)
|
|
#
|
|
# queue_uuid = data_dict['id']
|
|
# print(queue_uuid)
|
|
# dataBases = data_dict['dataBases']
|
|
# minSimilarity = data_dict['minSimilarity']
|
|
# minWords = data_dict['minWords']
|
|
# title = data_dict['title']
|
|
# author = data_dict['author']
|
|
# abst_zh = data_dict['abst_zh']
|
|
# content = data_dict['content']
|
|
# token = data_dict['token']
|
|
# account = data_dict['account']
|
|
# goodsId = data_dict['goodsId']
|
|
# callbackUrl = data_dict['callbackUrl']
|
|
#
|
|
# # 调用宇鹏查询相似十篇
|
|
# recall_data_list_dict = recall_10(queue_uuid, title, abst_zh, content)
|
|
#
|
|
# # print("查找相似的50篇完成")
|
|
# # with open("data/rell_json.txt") as f:
|
|
# # recall_data_list_dict = eval(f.read())
|
|
#
|
|
# # 读取文章转化成格式
|
|
|
|
|
|
def classify_accurate_check():
|
|
while True:
|
|
if redis_.llen(db_key_query_recall) == 0: # 若队列中没有元素就继续获取
|
|
time.sleep(3)
|
|
continue
|
|
|
|
query_recall = redis_.lpop(db_key_query_recall).decode('UTF-8') # 获取query的text
|
|
query_recall_dict = json.loads(query_recall) # db_key_query_recall json.dumps({"id": id_, "path": load_request_path})
|
|
query_recall_uuid = query_recall_dict["id"]
|
|
data_dict_path = query_recall_dict["path"]
|
|
print(data_dict_path)
|
|
|
|
# d = {
|
|
# "uuid_uesr": uuid_uesr,
|
|
# "content": content
|
|
# }
|
|
with open(data_dict_path, encoding='utf8') as f:
|
|
data_dict = json.loads(f.read())
|
|
|
|
|
|
uuid_uesr = data_dict['uuid_uesr']
|
|
abstract = data_dict['content'][0]
|
|
content = data_dict['content'][1]
|
|
# try:
|
|
recall_data_list = ulit_recall_paper(uuid_uesr)
|
|
|
|
print("查找相似的50篇完成")
|
|
|
|
# with open("data/rell_json.txt") as f:
|
|
# recall_data_list_dict = eval(f.read())
|
|
# recall_data_list = ulit_recall_paper(recall_data_list_dict)
|
|
|
|
print("文章格式转化完成")
|
|
|
|
# recall_data_list = pd.read_csv("data/10235513_大型商业建筑人员疏散设计研究_沈福禹/查重.csv", encoding="utf-8").values.tolist()
|
|
|
|
# 进入精确查重系统
|
|
print("进入精确查重系统")
|
|
|
|
return_list = accurate_check_rouge(content, recall_data_list)
|
|
|
|
|
|
return_text = {"resilt": return_list, "probabilities": None, "status_code": 200}
|
|
|
|
load_result_path = "./new_data_logs/{}.json".format(query_recall_uuid)
|
|
|
|
print("queue_uuid: ", query_recall_uuid)
|
|
print("load_result_path: ", load_result_path)
|
|
|
|
with open(load_result_path, 'w', encoding='utf8') as f2:
|
|
# ensure_ascii=False才能输入中文，否则是Unicode字符
|
|
# indent=2 JSON数据的缩进，美观
|
|
json.dump(return_text, f2, ensure_ascii=False, indent=4)
|
|
|
|
print(query_recall_uuid)
|
|
print(load_result_path)
|
|
redis_.set(query_recall_uuid, load_result_path, 86400)
|
|
redis_.srem(db_key_querying, query_recall_uuid)
|
|
# except:
|
|
# return_text = {"resilt": "", "probabilities": None, "status_code": 401}
|
|
# load_result_path = "./new_data_logs/{}.json".format(queue_uuid)
|
|
#
|
|
# print("queue_uuid: ", queue_uuid)
|
|
# print("load_result_path: ", load_result_path)
|
|
#
|
|
# with open(load_result_path, 'w', encoding='utf8') as f2:
|
|
# # ensure_ascii=False才能输入中文，否则是Unicode字符
|
|
# # indent=2 JSON数据的缩进，美观
|
|
# json.dump(return_text, f2, ensure_ascii=False, indent=4)
|
|
#
|
|
# print(queue_uuid)
|
|
# print(load_result_path)
|
|
# redis_.set(queue_uuid, load_result_path, 86400)
|
|
# redis_.srem(db_key_querying, queue_uuid)
|
|
|
|
|
|
@app.route("/", methods=["POST"])
|
|
def handle_query():
|
|
try:
|
|
print(request.remote_addr)
|
|
|
|
uuid_uesr = request.form.get("uuid")
|
|
file = request.files.get('file')
|
|
|
|
content = ulit_request_file(file)
|
|
|
|
id_ = str(uuid.uuid1()) # 为query生成唯一标识
|
|
# 绑定文本和query id
|
|
# recall_10(id_, title, abst_zh, content)
|
|
d = {
|
|
"uuid_uesr": uuid_uesr,
|
|
"content": content
|
|
}
|
|
load_request_path = './request_data_logs/{}.json'.format(id_)
|
|
with open(load_request_path, 'w', encoding='utf8') as f2:
|
|
json.dump(d, f2, ensure_ascii=False, indent=4)
|
|
|
|
redis_.rpush(db_key_query_recall, json.dumps({"id": id_, "path": load_request_path})) # 加入redis
|
|
return_text = {
|
|
'code': 0,
|
|
'msg': "请求成功",
|
|
'data': {
|
|
'balances': "",
|
|
'orderId': id_,
|
|
'consumeNum': ""
|
|
}
|
|
}
|
|
|
|
print("ok")
|
|
except:
|
|
return_text = {'code': 1}
|
|
return jsonify(return_text) # 返回结果
|
|
|
|
t1 = Thread(target=classify_accurate_check)
|
|
t1.start()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
app.run(host="0.0.0.0", port=20000, threaded=True)
|
|
|