diff --git a/flask_check.py b/flask_check.py
new file mode 100644
index 0000000..f35afd4
--- /dev/null
+++ b/flask_check.py
@@ -0,0 +1,474 @@
+import os
+import numpy as np
+from numpy.linalg import norm
+import pandas as pd
+# from rouge import Rouge
+from rouge_chinese import Rouge
+from Rouge_w import Rouge_w,Rouge_l
+import json
+import pymysql
+
+import requests
+from flask import Flask, jsonify
+from flask import request
+import uuid
+app = Flask(__name__)
+app.config["JSON_AS_ASCII"] = False
+
+
+nums_cpus = 16
+rouge = Rouge()
+rouge_model = Rouge_w()
+rouge_l_model = Rouge_l()
+
+
+def rouge_value_self(data_1, data_2):
+ data_1 = [' '.join(i) for i in data_1]
+ data_2 = [' '.join(i) for i in data_2]
+ rouge_l_list = []
+
+ for sen_1, sen_2 in zip(data_1, data_2):
+ sen_1 = sen_1.split(" ")
+ sen_2 = sen_2.split(" ")
+ rouge_l_score = rouge_l_model.score(sen_1, sen_2)
+ rouge_l_list.append(rouge_l_score)
+
+ return "", "", rouge_l_list
+
+
+def rouge_pre(text, df_train_nuoche):
+
+ return_list = []
+ index_rouge_list = []
+ text_list = [text] * len(df_train_nuoche)
+
+ data_list = []
+ for data_dan in df_train_nuoche:
+ data_list.append(data_dan[0])
+ rouge_1, rouge_2, rouge_l = rouge_value_self(text_list, data_list)
+ index_rouge_list.extend(rouge_l)
+
+ re1 = [(i[0], i[1]) for i in sorted(list(enumerate(index_rouge_list)), key=lambda x: x[1], reverse=True)]
+
+ return_list.append(re1[0][1])
+ return_list.append(re1[0][0])
+
+ return return_list
+
+
+def accurate_check_rouge(text_paper, recall_data_list):
+ '''
+ 精确查重出相似句子
+ :param text:
+ :param recall_data_list: list [[sentence, filename],[sentence, filename],[sentence, filename]]
+ :return:
+ '''
+ # 文本处理
+ # with open(text_paper_path, encoding="gbk") as f:
+ # text_paper = f.read()
+ centent_list = []
+ text_paper = str(text_paper).replace("。\n", "。")
+ centent_list.extend(text_paper.split("。"))
+ data_zong = []
+
+ # rouge算法查重
+ for text in centent_list:
+ rouge_pre_list = rouge_pre(text, recall_data_list)
+ data_zong.append(rouge_pre_list)
+
+ original_dict = []
+
+
+ # 找出相似的句子序号
+ bool_check_sentense = []
+ for i in range(len(data_zong)):
+ if data_zong[i][0] > 0.47:
+ bool_check_sentense.append([i,data_zong[i][1]])
+ biao_red = biaohong(bool_check_sentense, data_zong, recall_data_list) # [[[0, 1, 2], [479, 480, 481]], [[3, 4, 5], [481, 482, 483]], [[6, 7, 8], [484, 485, 486]]]
+
+ sentence_0_list = []
+ sentence_1_list = []
+ sim_paper_name = []
+
+ for i in biao_red:
+ if recall_data_list[i[1][0]][1] == recall_data_list[i[1][1]][1] == recall_data_list[i[1][2]][1]:
+ sentence_0_list.append("。".join([centent_list[i[0][0]], centent_list[i[0][1]], centent_list[i[0][2]]]))
+ sentence_1_list.append("".join([recall_data_list[i[1][0]][0], recall_data_list[i[1][1]][0], recall_data_list[i[1][2]][0]]))
+ sim_paper_name.append(recall_data_list[i[1][0]][1])
+ else:
+ continue
+
+ sentence_0_list_new = []
+ sentence_1_list_new = []
+
+ for i in zip(sentence_0_list, sentence_1_list):
+ if len(i[0]) + len(i[1]) < 1200:
+ sentence_0_list_new.append(i[0])
+ sentence_1_list_new.append(i[1])
+ else:
+ print(len(i[0]) + len(i[1]))
+ continue
+ for i in zip(sentence_0_list_new, sentence_1_list_new):
+ print("超过字数", len(i[0]))
+ print("超过字数", len(i[1]))
+
+ paper_dict = biaohong_bert_predict(sentence_0_list_new, sentence_1_list_new)
+
+ # paper_dict
+ # print("原文：".format(i), paper_dict[i][0])
+ # print("原文标红：".format(i), paper_dict[i][1])
+ # print("相似：".format(i), paper_dict[i][2])
+ # print("相似标红：".format(i), paper_dict[i][3])
+
+ # original_text
+ original_text = []
+ original_text_contrast = []
+
+
+ for paper_dict_dan_id, sentence_0_dan, sentence_1_dan, sim_paper_name_dan in zip(range(len(paper_dict)), sentence_0_list_new, sentence_1_list_new, sim_paper_name):
+
+ print([sentence_0_dan, sentence_1_dan])
+ original_text_contrast_dict = {}
+ similar_content = {"author": ""}
+ try:
+ sentence_0_bool, sentence_0_dan_red = original_text_marked_red(sentence_0_dan, paper_dict[paper_dict_dan_id][0], paper_dict[paper_dict_dan_id][1]) # text_original, bert_text, bert_text_pre
+ except:
+ print([sentence_0_dan,sentence_1_dan])
+ 9/0
+ sentence_1_bool, sentence_1_dan_red = original_text_marked_red(sentence_1_dan, paper_dict[paper_dict_dan_id][2], paper_dict[paper_dict_dan_id][3]) # text_original, bert_text, bert_text_pre
+
+ if sentence_0_bool == False or sentence_1_bool == False:
+ continue
+ original_text.append(sentence_0_dan_red)
+ original_text_contrast_dict["original_text"] = "此处有 {} 字相似\n".format(
+ len(paper_dict[paper_dict_dan_id][1])) + sentence_0_dan_red
+
+ similar_content["content"] = sentence_1_dan_red
+ similar_content["title"] = sim_paper_name_dan
+ original_text_contrast_dict["similar_content"] = similar_content
+
+ original_text_contrast.append(original_text_contrast_dict)
+
+ original_text = "。".join(original_text)
+
+ return {
+ "author": "",
+ "check_time": "",
+ "section_data": "",
+ "section_details": [
+ {
+ "end_page_index": 0,
+ "name": "",
+ "original_text": original_text,
+ "original_text_contrast": original_text_contrast
+ }
+ ]
+ }
+
+
+
+
+
+def biaohong(bool_check_sentense, data_zong, df_train_nuoche):
+ '''
+ 标红的序号 [[0,1,2],[3,4,5]]
+ :param bool_check_sentense:
+ :return: list
+ '''
+ biao_red = []
+ i = 0
+ start = -1
+ end = -1
+ while True:
+ if i >= len(bool_check_sentense) or bool_check_sentense[i][0] +1 >= len(data_zong) or bool_check_sentense[i][1]+1 >= len(df_train_nuoche):
+ break
+ elif bool_check_sentense[i][0]-1 == start:
+ i += 1
+ continue
+ elif bool_check_sentense[i][0] == end:
+ i += 1
+ continue
+ elif bool_check_sentense[i][0]-1 == end:
+ i += 1
+ continue
+ else:
+ biao_red_dan = []
+ biao_red_dan.append([bool_check_sentense[i][0] - 1, bool_check_sentense[i][1] - 1])
+ biao_red_dan.append([bool_check_sentense[i][0], bool_check_sentense[i][1]])
+ biao_red_dan.append([bool_check_sentense[i][0] + 1, bool_check_sentense[i][1] + 1])
+ biao_red.append([[bool_check_sentense[i][0]-1, bool_check_sentense[i][0], bool_check_sentense[i][0]+1],
+ [bool_check_sentense[i][1]-1, bool_check_sentense[i][1], bool_check_sentense[i][1]+1]])
+ start = bool_check_sentense[i][0]-1
+ end = bool_check_sentense[i][0]+1
+ i += 1
+
+ return biao_red
+
+
+def dialog_line_parse(url, text):
+ """
+ 将数据输入模型进行分析并输出结果
+ :param url: 模型url
+ :param text: 进入模型的数据
+ :return: 模型返回结果
+ """
+
+ response = requests.post(
+ url,
+ json=text,
+ timeout=100000
+ )
+ if response.status_code == 200:
+ return response.json()
+ else:
+ # logger.error(
+ # "【{}】 Failed to get a proper response from remote "
+ # "server. Status Code: {}. Response: {}"
+ # "".format(url, response.status_code, response.text)
+ # )
+ print("【{}】 Failed to get a proper response from remote "
+ "server. Status Code: {}. Response: {}"
+ "".format(url, response.status_code, response.text))
+ print(text)
+ return []
+
+
+def original_text_marked_red(text_original, bert_text, bert_text_pre):
+ '''
+ 把原文标红字段找到
+ :param text_original:
+ :param bert_text:
+ :param bert_text_pre:
+ :return:
+ '''
+
+ fuhao = ["\n"]
+ up_pointer = 0
+ down_pointer = 0
+
+ pointer_list = []
+
+ if len(bert_text_pre) > len(bert_text):
+ return False, ""
+
+ while True:
+ if down_pointer >= len(bert_text_pre):
+ break
+ elif down_pointer == len(bert_text_pre)-1:
+ if bert_text[up_pointer] == bert_text_pre[down_pointer]:
+ pointer_list.append(up_pointer)
+ break
+ else:
+ up_pointer += 1
+ down_pointer = 0
+ pointer_list = []
+
+ elif bert_text[up_pointer] in fuhao:
+ up_pointer += 1
+
+ else:
+ if bert_text[up_pointer] == bert_text_pre[down_pointer]:
+ pointer_list.append(up_pointer)
+ up_pointer += 1
+ down_pointer += 1
+ else:
+ if bert_text_pre[down_pointer:down_pointer+5] == "[UNK]":
+ up_pointer += 1
+ down_pointer += 5
+ pointer_list.append(up_pointer)
+ else:
+ up_pointer += 1
+ down_pointer = 0
+ pointer_list = []
+
+
+ start = pointer_list[0]
+ end = pointer_list[-1]
+ bert_text_list = list(bert_text)
+ bert_text_list.insert(start, "")
+ bert_text_list.insert(end + 2 , "")
+
+ text_original_list = list(text_original)
+
+ up = 0
+ down = 0
+
+ while True:
+ if up == len(text_original_list):
+ break
+
+ if text_original_list[up] == bert_text_list[down]:
+ up += 1
+ down += 1
+
+ else:
+ if bert_text_list[down] == "":
+ down += 1
+ elif bert_text_list[down] == "":
+ down += 1
+ else:
+ bert_text_list.insert(down, text_original_list[up])
+ up += 1
+ down += 1
+
+ bert_text = "".join(bert_text_list)
+ return True, bert_text
+
+
+def biaohong_bert_predict(sentence_0_list, sentence_1_list):
+ '''
+ 找出标红字符
+ :param bool_check_sentense:
+ :return:
+ '''
+
+ # sentence_0_list = []
+ # sentence_1_list = []
+ # sim_paper_name = []
+ #
+ # for i in biaohong_list:
+ # sentence_0_list.append("。".join([paper_list[i[0][0]], paper_list[i[0][1]], paper_list[i[0][2]]]))
+ # sentence_1_list.append("。".join([recall_data_list[i[1][1]], recall_data_list[i[1][1]], recall_data_list[i[1][2]]]))
+
+ paper_dict = dialog_line_parse("http://192.168.31.74:16003/", {"sentence_0": sentence_0_list, "sentence_1": sentence_1_list})["resilt"]
+
+ # paper_dict
+ # print("原文：".format(i), paper_dict[i][0])
+ # print("原文标红：".format(i), paper_dict[i][1])
+ # print("相似：".format(i), paper_dict[i][2])
+ # print("相似标红：".format(i), paper_dict[i][3])
+
+ # original_text
+ #
+ #
+ # for paper_dict_dan, sentence_0_dan, sentence_1_dan in zip(paper_dict, sentence_0_list, sentence_1_list):
+ # original_text_marked_red
+
+ return paper_dict
+
+def ulit_text(title, text):
+ data = []
+ try:
+ text = json.loads(text)["content"]
+ except:
+ pass
+
+ text = text.strip().replace("\n", "").replace(" ", "").replace("。", "。\n")
+ text_list = text.split("\n")
+
+ for i in text_list:
+ data.append([i, title])
+ return data
+
+def run_query(conn, sql, params):
+ with conn.cursor() as cursor:
+ cursor.execute(sql, params)
+ result = cursor.fetchall()
+ return result
+
+
+def processing_one_text(paper_id):
+ conn = pymysql.connect(
+ host='192.168.31.145',
+ port=3306,
+ user='root',
+ password='123456',
+ db='zhiwang_db',
+ charset='utf8mb4',
+ cursorclass=pymysql.cursors.DictCursor
+ )
+
+ sql = 'SELECT * FROM main_table_paper_detail_message WHERE doc_id=%s'
+ params = (paper_id,)
+
+ result = run_query(conn, sql, params)
+
+ conn.close()
+ print(result)
+ title = result[0]['title']
+ author = result[0]['author']
+ content_path = result[0]['content']
+
+ try:
+ with open(content_path, encoding="utf-8") as f:
+ text = f.read()
+ except:
+ with open(content_path, encoding="gbk") as f:
+ text = f.read()
+
+ data = ulit_text(title, text)
+ return data
+
+
+def ulit_recall_paper(recall_data_list_dict):
+ '''
+ 对返回的十篇文章路径读取并解析
+ :param recall_data_list_path:
+ :return data: list [[sentence, filename],[sentence, filename],[sentence, filename]]
+ '''
+
+ # data = []
+ # for path in recall_data_list_path:
+ # filename = path.split("/")[-1]
+ # with open(path, encoding="gbk") as f:
+ # text = f.read()
+ # text_list = text.split("\n")
+ # for sentence in text_list:
+ # if sentence != "":
+ # data.append([sentence, filename])
+ # return data
+
+
+ data = []
+ for i in list(recall_data_list_dict.items())[:5]:
+ data_one = processing_one_text(i[0])
+ data.extend(data_one)
+
+
+ return data
+
+
+def recall_10(title, abst_zh, content) -> list:
+ '''
+ 宇鹏召回接口
+ :param paper_name:
+ :return:
+ '''
+
+ request_json = {
+ "title": title,
+ "abst_zh": abst_zh,
+ "content": content
+ }
+ paper_dict = dialog_line_parse("http://192.168.31.145:50002/check", request_json)
+
+ return paper_dict
+
+@app.route("/", methods=["POST"])
+def handle_query():
+
+ title = request.json["title"]
+ abst_zh = request.json["abst_zh"] # txt
+ content = request.json["content"]
+
+ # 调用宇鹏查询相似十篇
+ # recall_data_list_dict = recall_10(title, abst_zh, content)
+
+ with open("data/rell_json.txt") as f:
+ recall_data_list_dict = eval(f.read())
+
+ # 读取文章转化成格式数据
+ recall_data_list = ulit_recall_paper(recall_data_list_dict)
+
+
+ # recall_data_list = pd.read_csv("data/10235513_大型商业建筑人员疏散设计研究_沈福禹/查重.csv", encoding="utf-8").values.tolist()
+
+ # 进入精确查重系统
+ return_list = accurate_check_rouge(content, recall_data_list)
+
+ return_text = {"resilt": return_list, "probabilities": None, "status_code": 200}
+ return jsonify(return_text) # 返回结果
+
+
+if __name__ == "__main__":
+ app.run(host="0.0.0.0", port=16001, threaded=True, debug=True)
\ No newline at end of file
diff --git a/flask_check_bert.py b/flask_check_bert.py
new file mode 100644
index 0000000..2a6010e
--- /dev/null
+++ b/flask_check_bert.py
@@ -0,0 +1,523 @@
+import os
+import numpy as np
+from numpy.linalg import norm
+import pandas as pd
+# from rouge import Rouge
+from rouge_chinese import Rouge
+from Rouge_w import Rouge_w,Rouge_l
+import json
+import pymysql
+
+import requests
+from flask import Flask, jsonify
+from flask import request
+import uuid
+app = Flask(__name__)
+app.config["JSON_AS_ASCII"] = False
+
+
+nums_cpus = 16
+rouge = Rouge()
+rouge_model = Rouge_w()
+rouge_l_model = Rouge_l()
+
+
+def bert_check(text, recall_data_list):
+ '''
+ bert 查重
+ :return:
+ '''
+
+ sen_0 = [text] * len(recall_data_list)
+ sen_1 = [i[0] for i in recall_data_list]
+
+ return_list = []
+ request_json = {
+ "texts": [sen_0, sen_1],
+ }
+ paper_dict = dialog_line_parse("http://192.168.31.74:16002/", request_json)
+ score_list = paper_dict["res"]
+
+ # 后期要改
+ # return_list.append(re1[0][1])
+ # return_list.append(re1[0][0])
+ if 1 in score_list:
+ index_score = score_list.index(1)
+ else:
+ index_score = "NaN"
+
+ if index_score == "NaN":
+ return_list.append(0)
+ return_list.append("")
+ else:
+ return_list.append(1)
+ return_list.append(index_score)
+
+ return return_list
+
+
+
+def rouge_value_self(data_1, data_2):
+ data_1 = [' '.join(i) for i in data_1]
+ data_2 = [' '.join(i) for i in data_2]
+ rouge_l_list = []
+
+ for sen_1, sen_2 in zip(data_1, data_2):
+ sen_1 = sen_1.split(" ")
+ sen_2 = sen_2.split(" ")
+ rouge_l_score = rouge_l_model.score(sen_1, sen_2)
+ rouge_l_list.append(rouge_l_score)
+
+ return "", "", rouge_l_list
+
+
+def rouge_pre(text, df_train_nuoche):
+
+ return_list = []
+ index_rouge_list = []
+ text_list = [text] * len(df_train_nuoche)
+
+ data_list = []
+ for data_dan in df_train_nuoche:
+ data_list.append(data_dan[0])
+ rouge_1, rouge_2, rouge_l = rouge_value_self(text_list, data_list)
+ index_rouge_list.extend(rouge_l)
+
+ re1 = [(i[0], i[1]) for i in sorted(list(enumerate(index_rouge_list)), key=lambda x: x[1], reverse=True)]
+
+ return_list.append(re1[0][1])
+ return_list.append(re1[0][0])
+
+ return return_list
+
+
+def accurate_check_rouge(text_paper, recall_data_list):
+ '''
+ 精确查重出相似句子
+ :param text:
+ :param recall_data_list: list [[sentence, filename],[sentence, filename],[sentence, filename]]
+ :return:
+ '''
+ # 文本处理
+ # with open(text_paper_path, encoding="gbk") as f:
+ # text_paper = f.read()
+ centent_list = []
+ text_paper = str(text_paper).replace("。\n", "。")
+ centent_list.extend(text_paper.split("。"))
+ data_zong = []
+
+ # rouge算法查重
+ # for text in centent_list:
+ # rouge_pre_list = rouge_pre(text, recall_data_list)
+ # data_zong.append(rouge_pre_list)
+
+ # bert算法查重
+ for text in centent_list:
+ bert_pre_list = bert_check(text, recall_data_list)
+ data_zong.append(bert_pre_list)
+
+ original_dict = []
+
+
+ # 找出相似的句子序号
+ bool_check_sentense = []
+ for i in range(len(data_zong)):
+ if data_zong[i][0] == 1:
+ bool_check_sentense.append([i,data_zong[i][1]])
+ biao_red = biaohong(bool_check_sentense, data_zong, recall_data_list) # [[[0, 1, 2], [479, 480, 481]], [[3, 4, 5], [481, 482, 483]], [[6, 7, 8], [484, 485, 486]]]
+
+ sentence_0_list = []
+ sentence_1_list = []
+ sim_paper_name = []
+
+ for i in biao_red:
+ if recall_data_list[i[1][0]][1] == recall_data_list[i[1][1]][1] == recall_data_list[i[1][2]][1]:
+ sentence_0_list.append("。".join([centent_list[i[0][0]], centent_list[i[0][1]], centent_list[i[0][2]]]))
+ sentence_1_list.append("".join([recall_data_list[i[1][0]][0], recall_data_list[i[1][1]][0], recall_data_list[i[1][2]][0]]))
+ sim_paper_name.append(recall_data_list[i[1][0]][1])
+ else:
+ continue
+
+ sentence_0_list_new = []
+ sentence_1_list_new = []
+
+
+ for i in zip(sentence_0_list, sentence_1_list):
+ if len(i[0]) + len(i[1]) < 1200:
+ sentence_0_list_new.append(i[0])
+ sentence_1_list_new.append(i[1])
+ else:
+ print(len(i[0]) + len(i[1]))
+ continue
+ for i in zip(sentence_0_list_new, sentence_1_list_new):
+ print("超过字数", len(i[0]))
+ print("超过字数", len(i[1]))
+
+ paper_dict = biaohong_bert_predict(sentence_0_list_new, sentence_1_list_new)
+
+ # paper_dict
+ # print("原文：".format(i), paper_dict[i][0])
+ # print("原文标红：".format(i), paper_dict[i][1])
+ # print("相似：".format(i), paper_dict[i][2])
+ # print("相似标红：".format(i), paper_dict[i][3])
+
+ # original_text
+ original_text = []
+ original_text_contrast = []
+
+
+ for paper_dict_dan_id, sentence_0_dan, sentence_1_dan, sim_paper_name_dan in zip(range(len(paper_dict)), sentence_0_list_new, sentence_1_list_new, sim_paper_name):
+
+ print([sentence_0_dan, sentence_1_dan])
+ original_text_contrast_dict = {}
+ similar_content = {"author": ""}
+ try:
+ sentence_0_bool, sentence_0_dan_red = original_text_marked_red(sentence_0_dan, paper_dict[paper_dict_dan_id][0], paper_dict[paper_dict_dan_id][1]) # text_original, bert_text, bert_text_pre
+ except:
+ print("报错", [sentence_0_dan, paper_dict[paper_dict_dan_id][0], paper_dict[paper_dict_dan_id][1]])
+ 9/0
+ sentence_1_bool, sentence_1_dan_red = original_text_marked_red(sentence_1_dan, paper_dict[paper_dict_dan_id][2], paper_dict[paper_dict_dan_id][3]) # text_original, bert_text, bert_text_pre
+
+ if sentence_0_bool == False or sentence_1_bool == False:
+ continue
+ original_text.append(sentence_0_dan_red)
+ original_text_contrast_dict["original_text"] = "此处有 {} 字相似\n".format(
+ len(paper_dict[paper_dict_dan_id][1])) + sentence_0_dan_red
+
+ similar_content["content"] = sentence_1_dan_red
+ similar_content["title"] = sim_paper_name_dan
+ original_text_contrast_dict["similar_content"] = similar_content
+
+ original_text_contrast.append(original_text_contrast_dict)
+
+ original_text = "。".join(original_text)
+
+ return {
+ "author": "",
+ "check_time": "",
+ "section_data": "",
+ "section_details": [
+ {
+ "end_page_index": 0,
+ "name": "",
+ "original_text": original_text,
+ "original_text_contrast": original_text_contrast
+ }
+ ]
+ }
+
+
+
+
+
+def biaohong(bool_check_sentense, data_zong, df_train_nuoche):
+ '''
+ 标红的序号 [[0,1,2],[3,4,5]]
+ :param bool_check_sentense:
+ :return: list
+ '''
+ biao_red = []
+ i = 0
+ start = -1
+ end = -1
+ while True:
+ if i >= len(bool_check_sentense) or bool_check_sentense[i][0] +1 >= len(data_zong) or bool_check_sentense[i][1]+1 >= len(df_train_nuoche):
+ break
+ elif bool_check_sentense[i][0]-1 == start:
+ i += 1
+ continue
+ elif bool_check_sentense[i][0] == end:
+ i += 1
+ continue
+ elif bool_check_sentense[i][0]-1 == end:
+ i += 1
+ continue
+ else:
+ biao_red_dan = []
+ biao_red_dan.append([bool_check_sentense[i][0] - 1, bool_check_sentense[i][1] - 1])
+ biao_red_dan.append([bool_check_sentense[i][0], bool_check_sentense[i][1]])
+ biao_red_dan.append([bool_check_sentense[i][0] + 1, bool_check_sentense[i][1] + 1])
+ biao_red.append([[bool_check_sentense[i][0]-1, bool_check_sentense[i][0], bool_check_sentense[i][0]+1],
+ [bool_check_sentense[i][1]-1, bool_check_sentense[i][1], bool_check_sentense[i][1]+1]])
+ start = bool_check_sentense[i][0]-1
+ end = bool_check_sentense[i][0]+1
+ i += 1
+
+ return biao_red
+
+
+def dialog_line_parse(url, text):
+ """
+ 将数据输入模型进行分析并输出结果
+ :param url: 模型url
+ :param text: 进入模型的数据
+ :return: 模型返回结果
+ """
+
+ response = requests.post(
+ url,
+ json=text,
+ timeout=100000
+ )
+ if response.status_code == 200:
+ return response.json()
+ else:
+ # logger.error(
+ # "【{}】 Failed to get a proper response from remote "
+ # "server. Status Code: {}. Response: {}"
+ # "".format(url, response.status_code, response.text)
+ # )
+ print("【{}】 Failed to get a proper response from remote "
+ "server. Status Code: {}. Response: {}"
+ "".format(url, response.status_code, response.text))
+ print(text)
+ return []
+
+
+def is_english_char(char):
+ code = ord(char)
+ return 32 <= code <= 126
+
+
+def original_text_marked_red(text_original, bert_text, bert_text_pre):
+ '''
+ 把原文标红字段找到
+ :param text_original:
+ :param bert_text:
+ :param bert_text_pre:
+ :return:
+ '''
+
+ fuhao = ["\n"]
+ up_pointer = 0
+ down_pointer = 0
+
+ pointer_list = []
+
+ if len(bert_text_pre) > len(bert_text):
+ return False, ""
+
+ while True:
+ if down_pointer >= len(bert_text_pre):
+ break
+ elif down_pointer == len(bert_text_pre)-1:
+ if bert_text[up_pointer] == bert_text_pre[down_pointer]:
+ pointer_list.append(up_pointer)
+ break
+ else:
+ up_pointer += 1
+ down_pointer = 0
+ pointer_list = []
+
+ elif bert_text[up_pointer] in fuhao:
+ up_pointer += 1
+
+ else:
+ if bert_text[up_pointer] == bert_text_pre[down_pointer]:
+ pointer_list.append(up_pointer)
+ up_pointer += 1
+ down_pointer += 1
+ else:
+ if bert_text_pre[down_pointer:down_pointer+5] == "[UNK]":
+ up_pointer += 1
+ down_pointer += 5
+ pointer_list.append(up_pointer)
+ elif is_english_char(bert_text_pre[down_pointer]) == True:
+ up_pointer += 1
+ down_pointer += 1
+ pointer_list.append(up_pointer)
+ else:
+ up_pointer += 1
+ down_pointer = 0
+ pointer_list = []
+
+
+ start = pointer_list[0]
+ end = pointer_list[-1]
+ bert_text_list = list(bert_text)
+ bert_text_list.insert(start, "")
+ bert_text_list.insert(end + 2 , "")
+
+ text_original_list = list(text_original)
+
+ up = 0
+ down = 0
+
+ while True:
+ if up == len(text_original_list):
+ break
+
+ if text_original_list[up] == bert_text_list[down]:
+ up += 1
+ down += 1
+
+ else:
+ if bert_text_list[down] == "":
+ down += 1
+ elif bert_text_list[down] == "":
+ down += 1
+ else:
+ bert_text_list.insert(down, text_original_list[up])
+ up += 1
+ down += 1
+
+ bert_text = "".join(bert_text_list)
+ return True, bert_text
+
+
+def biaohong_bert_predict(sentence_0_list, sentence_1_list):
+ '''
+ 找出标红字符
+ :param bool_check_sentense:
+ :return:
+ '''
+
+ # sentence_0_list = []
+ # sentence_1_list = []
+ # sim_paper_name = []
+ #
+ # for i in biaohong_list:
+ # sentence_0_list.append("。".join([paper_list[i[0][0]], paper_list[i[0][1]], paper_list[i[0][2]]]))
+ # sentence_1_list.append("。".join([recall_data_list[i[1][1]], recall_data_list[i[1][1]], recall_data_list[i[1][2]]]))
+
+ paper_dict = dialog_line_parse("http://192.168.31.74:16003/", {"sentence_0": sentence_0_list, "sentence_1": sentence_1_list})["resilt"]
+
+ # paper_dict
+ # print("原文：".format(i), paper_dict[i][0])
+ # print("原文标红：".format(i), paper_dict[i][1])
+ # print("相似：".format(i), paper_dict[i][2])
+ # print("相似标红：".format(i), paper_dict[i][3])
+
+ # original_text
+ #
+ #
+ # for paper_dict_dan, sentence_0_dan, sentence_1_dan in zip(paper_dict, sentence_0_list, sentence_1_list):
+ # original_text_marked_red
+
+ return paper_dict
+
+def ulit_text(title, text):
+ data = []
+ try:
+ text = json.loads(text)["content"]
+ except:
+ pass
+
+ text = text.strip().replace("\n", "").replace(" ", "").replace("。", "。\n")
+ text_list = text.split("\n")
+
+ for i in text_list:
+ data.append([i, title])
+ return data
+
+def run_query(conn, sql, params):
+ with conn.cursor() as cursor:
+ cursor.execute(sql, params)
+ result = cursor.fetchall()
+ return result
+
+
+def processing_one_text(paper_id):
+ conn = pymysql.connect(
+ host='192.168.31.145',
+ port=3306,
+ user='root',
+ password='123456',
+ db='zhiwang_db',
+ charset='utf8mb4',
+ cursorclass=pymysql.cursors.DictCursor
+ )
+
+ sql = 'SELECT * FROM main_table_paper_detail_message WHERE doc_id=%s'
+ params = (paper_id,)
+
+ result = run_query(conn, sql, params)
+
+ conn.close()
+ print(result)
+ title = result[0]['title']
+ author = result[0]['author']
+ content_path = result[0]['content']
+
+ try:
+ with open(content_path, encoding="utf-8") as f:
+ text = f.read()
+ except:
+ with open(content_path, encoding="gbk") as f:
+ text = f.read()
+
+ data = ulit_text(title, text)
+ return data
+
+
+def ulit_recall_paper(recall_data_list_dict):
+ '''
+ 对返回的十篇文章路径读取并解析
+ :param recall_data_list_path:
+ :return data: list [[sentence, filename],[sentence, filename],[sentence, filename]]
+ '''
+
+ # data = []
+ # for path in recall_data_list_path:
+ # filename = path.split("/")[-1]
+ # with open(path, encoding="gbk") as f:
+ # text = f.read()
+ # text_list = text.split("\n")
+ # for sentence in text_list:
+ # if sentence != "":
+ # data.append([sentence, filename])
+ # return data
+
+
+ data = []
+ for i in list(recall_data_list_dict.items())[:5]:
+ data_one = processing_one_text(i[0])
+ data.extend(data_one)
+
+ return data
+
+
+def recall_10(title, abst_zh, content) -> list:
+ '''
+ 宇鹏召回接口
+ :param paper_name:
+ :return:
+ '''
+
+ request_json = {
+ "title": title,
+ "abst_zh": abst_zh,
+ "content": content
+ }
+ paper_dict = dialog_line_parse("http://192.168.31.145:50002/check", request_json)
+
+ return paper_dict
+
+@app.route("/", methods=["POST"])
+def handle_query():
+ print(request.remote_addr)
+ title = request.json["title"]
+ abst_zh = request.json["abst_zh"] # txt
+ content = request.json["content"]
+
+ # 调用宇鹏查询相似十篇
+ # recall_data_list_dict = recall_10(title, abst_zh, content)
+ with open("data/rell_json.txt") as f:
+ recall_data_list_dict = eval(f.read())
+
+
+ # 读取文章转化成格式数据
+ recall_data_list = ulit_recall_paper(recall_data_list_dict)
+
+
+ # recall_data_list = pd.read_csv("data/10235513_大型商业建筑人员疏散设计研究_沈福禹/查重.csv", encoding="utf-8").values.tolist()
+
+ # 进入精确查重系统
+ return_list = accurate_check_rouge(content, recall_data_list)
+
+ return_text = {"resilt": return_list, "probabilities": None, "status_code": 200}
+ return jsonify(return_text) # 返回结果
+
+
+if __name__ == "__main__":
+ app.run(host="0.0.0.0", port=16001, threaded=True, debug=True)
\ No newline at end of file