From 6e9434987e4106bf56c94ad96b27544ba4882bac Mon Sep 17 00:00:00 2001 From: "majiahui@haimaqingfan.com" Date: Tue, 29 Aug 2023 14:49:27 +0800 Subject: [PATCH] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E5=A4=9A=E8=BF=9B=E7=A8=8B?= =?UTF-8?q?=E6=A3=80=E6=B5=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- flask_check_bert_test.py | 903 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 903 insertions(+) create mode 100644 flask_check_bert_test.py diff --git a/flask_check_bert_test.py b/flask_check_bert_test.py new file mode 100644 index 0000000..c40ff48 --- /dev/null +++ b/flask_check_bert_test.py @@ -0,0 +1,903 @@ +import os +import numpy as np +from numpy.linalg import norm +import pandas as pd +# from rouge import Rouge +from rouge_chinese import Rouge +from Rouge_w import Rouge_w,Rouge_l +import json +import pymysql +import re +import requests +from flask import Flask, jsonify +from flask import request +import uuid +import time +import redis +from threading import Thread +from multiprocessing import Pool +app = Flask(__name__) +app.config["JSON_AS_ASCII"] = False + +pool = redis.ConnectionPool(host='localhost', port=63179, max_connections=100, db=7, password="zhicheng123*") +redis_ = redis.Redis(connection_pool=pool, decode_responses=True) + +db_key_query = 'query' +db_key_querying = 'querying' +db_key_queryset = 'queryset' + +nums_cpus = 24 +rouge = Rouge() +rouge_model = Rouge_w() +rouge_l_model = Rouge_l() + + +def bert_check(text, recall_data_list): + ''' + bert 查重 + :return: + ''' + + sen_0 = [text] * len(recall_data_list) + sen_1 = [i[0] for i in recall_data_list] + + return_list = [] + request_json = { + "texts": [sen_0, sen_1], + } + paper_dict = dialog_line_parse("http://192.168.31.74:16002/", request_json) + score_list = paper_dict["res"] + + # 后期要改 + # return_list.append(re1[0][1]) + # return_list.append(re1[0][0]) + if 1 in score_list: + index_score = score_list.index(1) + else: + index_score = "NaN" + + if index_score == "NaN": + return_list.append(0) + return_list.append("") + else: + return_list.append(1) + return_list.append(index_score) + + return return_list + + + +def rouge_value_self(data_1, data_2): + data_1 = [' '.join(i) for i in data_1] + data_2 = [' '.join(i) for i in data_2] + rouge_l_list = [] + + for sen_1, sen_2 in zip(data_1, data_2): + sen_1 = sen_1.split(" ") + sen_2 = sen_2.split(" ") + rouge_l_score = rouge_l_model.score(sen_1, sen_2) + rouge_l_list.append(rouge_l_score) + + return "", "", rouge_l_list + + +def rouge_pre(text, df_train_nuoche): + + return_list = [] + index_rouge_list = [] + text_list = [text] * len(df_train_nuoche) + + data_list = [] + for data_dan in df_train_nuoche: + data_list.append(data_dan[0]) + rouge_1, rouge_2, rouge_l = rouge_value_self(text_list, data_list) + index_rouge_list.extend(rouge_l) + + re1 = [(i[0], i[1]) for i in sorted(list(enumerate(index_rouge_list)), key=lambda x: x[1], reverse=True)] + + return_list.append(re1[0][1]) + return_list.append(re1[0][0]) + + return return_list + + +def rouge_pre_m(text, df_train_nuoche): + + return_list = [] + index_rouge_list = [] + + text_list = [text] * len(df_train_nuoche) + + data_list = [] + for data_dan in df_train_nuoche: + data_list.append(data_dan[0]) + rouge_1, rouge_2, rouge_l = rouge_value_self(text_list, data_list) + index_rouge_list.extend(rouge_l) + + re1 = [(i[0], i[1]) for i in sorted(list(enumerate(index_rouge_list)), key=lambda x: x[1], reverse=True)] + + return_list.append(re1[0][1]) + return_list.append(re1[0][0]) + + return return_list + + +def accurate_check_rouge( + title, + author, + text_paper, + recall_data_list + ): + ''' + 精确查重出相似句子 + :param text: + :param recall_data_list: list [[sentence, filename],[sentence, filename],[sentence, filename]] + :return: + ''' + # 文本处理 + centent_list = [] + text_paper = str(text_paper).replace("。\n", "。") + centent_list.extend(text_paper.split("。")) + data_zong = [] + sentence_word_nums = 0 + + # rouge算法查重 + rst = [] + p = Pool(nums_cpus) # 进程池中含有n个子进程 + + print("centent_list", centent_list) + + for i in range(len(centent_list)): + text = centent_list[i] + a = p.apply_async(rouge_pre_m, args=(text, recall_data_list,)) + rst.append(a) + p.close() + p.join() # 等待所有子进程执行完毕。调用join()之前必须先调用close()，调用close()之后就不能继续添加新的Process了。 + + rst = [i.get() for i in rst] + + for i in range(len(rst)): + print(rst[i]) + data_zong.append(rst[i]) + + t0 = time.time() + # bert算法查重 + # for text in centent_list: + # bert_pre_list = bert_check(text, recall_data_list) + # data_zong.append(bert_pre_list) + t1 = time.time() + original_dict = [] + + + # 找出相似的句子序号 + bool_check_sentense = [] + # bert算法 + # for i in range(len(data_zong)): + # if data_zong[i][0] == 1: + # bool_check_sentense.append([i,data_zong[i][1]]) + + # rouge算法 + for i in range(len(data_zong)): + if data_zong[i][0] > 0.47: + bool_check_sentense.append([i,data_zong[i][1]]) + biao_red = biaohong(bool_check_sentense, data_zong, recall_data_list) # [[[0, 1, 2], [479, 480, 481]], [[3, 4, 5], [481, 482, 483]], [[6, 7, 8], [484, 485, 486]]] + + print("bert精确查重时间", t1-t0) + + + sentence_0_list = [] + sentence_1_list = [] + sim_paper_name = [] + + for i in biao_red: + if recall_data_list[i[1][0]][1] == recall_data_list[i[1][1]][1] == recall_data_list[i[1][2]][1]: + sentence_0_list.append("。".join([centent_list[i[0][0]], centent_list[i[0][1]], centent_list[i[0][2]]])) + sentence_1_list.append("".join([recall_data_list[i[1][0]][0], recall_data_list[i[1][1]][0], recall_data_list[i[1][2]][0]])) + sim_paper_name.append(recall_data_list[i[1][0]][1]) + else: + continue + + sentence_0_list_new = [] + sentence_1_list_new = [] + + + for i in zip(sentence_0_list, sentence_1_list): + if len(i[0]) + len(i[1]) < 1200: + sentence_0_list_new.append(i[0]) + sentence_1_list_new.append(i[1]) + else: + print(len(i[0]) + len(i[1])) + continue + t2 = time.time() + paper_dict = biaohong_bert_predict(sentence_0_list_new, sentence_1_list_new) + + t3 = time.time() + print("标红时间", t3 - t2) + original_text = [] + original_text_contrast = [] + repeat_quote_info = [] + + chongfuwendang = {} + + for paper_dict_dan_id, sentence_0_dan, sentence_1_dan, sim_paper_name_dan in zip(range(len(paper_dict)), sentence_0_list_new, sentence_1_list_new, sim_paper_name): + + print([sentence_0_dan, sentence_1_dan]) + original_text_contrast_dict = { + "original_text": "", + "similar_content": [ + { + "content": "", + "thesis_info": "", + "title": "", + "year": "", + "degree": "", + "author": "", + } + ] + } + try: + sentence_0_bool, sentence_0_dan_red = original_text_marked_red(sentence_0_dan, paper_dict[paper_dict_dan_id][0], paper_dict[paper_dict_dan_id][1]) # text_original, bert_text, bert_text_pre + except: + print("报错", [sentence_0_dan, paper_dict[paper_dict_dan_id][0], paper_dict[paper_dict_dan_id][1]]) + continue + # 9/0 + sentence_1_bool, sentence_1_dan_red = original_text_marked_red(sentence_1_dan, paper_dict[paper_dict_dan_id][2], paper_dict[paper_dict_dan_id][3]) # text_original, bert_text, bert_text_pre + + if sentence_0_bool == False or sentence_1_bool == False: + continue + + dan_sentence_word_nums = len(paper_dict[paper_dict_dan_id][1]) + sentence_word_nums += dan_sentence_word_nums + + original_text.append(sentence_0_dan_red) + original_text_contrast_dict["original_text"] = "此处有 {} 字相似\n".format( + dan_sentence_word_nums) + sentence_0_dan_red + + thesis_info = " ".join([sim_paper_name_dan["title"], sim_paper_name_dan["author"], sim_paper_name_dan["degree"], sim_paper_name_dan["year"]]) + original_text_contrast_dict["similar_content"][0]["content"] = sentence_1_dan_red + original_text_contrast_dict["similar_content"][0]["title"] = sim_paper_name_dan["title"] + original_text_contrast_dict["similar_content"][0]["author"] = sim_paper_name_dan["author"] + original_text_contrast_dict["similar_content"][0]["degree"] = sim_paper_name_dan["degree"] + original_text_contrast_dict["similar_content"][0]["year"] = sim_paper_name_dan["year"] + original_text_contrast_dict["similar_content"][0]["thesis_info"] = thesis_info + + original_text_contrast.append(original_text_contrast_dict) + + # for i in repeat_quote_info: + # if + + if thesis_info not in chongfuwendang: + chongfuwendang[thesis_info] = { + "quote": False, + "thesis_author": sim_paper_name_dan["author"], + "thesis_date" : sim_paper_name_dan["year"], + "thesis_info" : thesis_info, + "thesis_repeat_rate": (dan_sentence_word_nums/sim_paper_name_dan["paper_len_word"]) * 100, #round(repetition_rate, 3) * 100 + "thesis_title": sim_paper_name_dan["title"], + "thesis_link": "", + "thesis_publish": sim_paper_name_dan["degree"], + "thesis_repeat_word": dan_sentence_word_nums, + "thesis_teacher": "", + "paper_len_word": sim_paper_name_dan["paper_len_word"] + } + else: + chongfuwendang[thesis_info]["thesis_repeat_word"] += dan_sentence_word_nums + chongfuwendang[thesis_info]["thesis_repeat_rate"] = (chongfuwendang[thesis_info]["thesis_repeat_word"]/chongfuwendang[thesis_info]["paper_len_word"]) * 100 + + + chongfuwendang = sorted(chongfuwendang.items(), + key=lambda x: x[1]["thesis_repeat_rate"], reverse=False) + + + for i in range(len(chongfuwendang)): + repeat_paper_one_info_dict = chongfuwendang[i][1] + repeat_paper_one_info_dict.pop("paper_len_word") + repeat_paper_one_info_dict["thesis_repeat_rate"] = str(round(repeat_paper_one_info_dict["thesis_repeat_rate"], 1)) + "%" + repeat_quote_info.append(repeat_paper_one_info_dict) + + original_text = "。".join(original_text) + + repetition_rate = sentence_word_nums/len(text_paper) + repetition_rate = round(repetition_rate, 3) * 100 + + format = '%Y-%m-%d %H:%M:%S' + value = time.localtime(int(time.time())) + dt = time.strftime(format, value) + + return { + "author": author, + "check_time": dt, + "title": title, + "time_range": "1900-01-01至2023-08-08", + "section_data": [ + { + "oneself_repeat_words": sentence_word_nums, + "reference_repeat_words": sentence_word_nums, + "section_name": "第1部分", + "section_oneself_rate": "{}%".format(repetition_rate), + "section_repeat_rate": "{}%".format(repetition_rate), + "section_repeat_words": sentence_word_nums, + "section_words": len(text_paper) + } + ], + "section_details": [ + { + "end_page_index": 0, + "name": "", + "repeat_rate": "", + "repeat_words": "", + "words": "", + "original_text": original_text, + "original_text_oneself": original_text, + "original_text_contrast": original_text_contrast, + "repeat_quote_info": repeat_quote_info + } + ], + "total_data": { + "back_repeat_words": "", + "exclude_personal_rate": "{}%".format(repetition_rate), + "exclude_quote_rate": "{}%".format(repetition_rate), + "foot_end_note": "0", + "front_repeat_words": "", + "single_max_rate": "", + "single_max_repeat_words": "", + "suspected_paragraph": "1", + "suspected_paragraph_max_repeat_words": "", + "suspected_paragraph_min_repeat_words": "", + "tables": "0", + "total_paragraph": "1", + "total_repeat_rate": "{}%".format(repetition_rate), + "total_repeat_words": sentence_word_nums, + "total_words": len(text_paper) + } + } + + + + + +def biaohong(bool_check_sentense, data_zong, df_train_nuoche): + ''' + 标红的序号 [[0,1,2],[3,4,5]] + :param bool_check_sentense: + :return: list + ''' + biao_red = [] + i = 0 + start = -1 + end = -1 + while True: + if i >= len(bool_check_sentense) or bool_check_sentense[i][0] +1 >= len(data_zong) or bool_check_sentense[i][1]+1 >= len(df_train_nuoche): + break + elif bool_check_sentense[i][0]-1 == start: + i += 1 + continue + elif bool_check_sentense[i][0] == end: + i += 1 + continue + elif bool_check_sentense[i][0]-1 == end: + i += 1 + continue + else: + biao_red_dan = [] + biao_red_dan.append([bool_check_sentense[i][0] - 1, bool_check_sentense[i][1] - 1]) + biao_red_dan.append([bool_check_sentense[i][0], bool_check_sentense[i][1]]) + biao_red_dan.append([bool_check_sentense[i][0] + 1, bool_check_sentense[i][1] + 1]) + biao_red.append([[bool_check_sentense[i][0]-1, bool_check_sentense[i][0], bool_check_sentense[i][0]+1], + [bool_check_sentense[i][1]-1, bool_check_sentense[i][1], bool_check_sentense[i][1]+1]]) + start = bool_check_sentense[i][0]-1 + end = bool_check_sentense[i][0]+1 + i += 1 + + return biao_red + + +def dialog_line_parse(url, text): + """ + 将数据输入模型进行分析并输出结果 + :param url: 模型url + :param text: 进入模型的数据 + :return: 模型返回结果 + """ + + response = requests.post( + url, + json=text, + timeout=100000 + ) + if response.status_code == 200: + return response.json() + else: + # logger.error( + # "【{}】 Failed to get a proper response from remote " + # "server. Status Code: {}. Response: {}" + # "".format(url, response.status_code, response.text) + # ) + print("【{}】 Failed to get a proper response from remote " + "server. Status Code: {}. Response: {}" + "".format(url, response.status_code, response.text)) + print(text) + return {} + + +def is_english_char(char): + code = ord(char) + return 32 <= code <= 126 + + +def original_text_marked_red(text_original, bert_text, bert_text_pre): + ''' + 把原文标红字段找到 + :param text_original: + :param bert_text: + :param bert_text_pre: + :return: + ''' + + fuhao = ["\n"] + up_pointer = 0 + down_pointer = 0 + + pointer_list = [] + + if len(bert_text_pre) > len(bert_text): + return False, "" + + while True: + if down_pointer >= len(bert_text_pre): + break + elif down_pointer == len(bert_text_pre)-1: + if bert_text[up_pointer] == bert_text_pre[down_pointer]: + pointer_list.append(up_pointer) + break + else: + up_pointer += 1 + down_pointer = 0 + pointer_list = [] + + elif bert_text[up_pointer] in fuhao: + up_pointer += 1 + + else: + if bert_text[up_pointer] == bert_text_pre[down_pointer]: + pointer_list.append(up_pointer) + up_pointer += 1 + down_pointer += 1 + else: + if bert_text_pre[down_pointer:down_pointer+5] == "[UNK]": + up_pointer += 1 + down_pointer += 5 + pointer_list.append(up_pointer) + elif is_english_char(bert_text_pre[down_pointer]) == True: + up_pointer += 1 + down_pointer += 1 + pointer_list.append(up_pointer) + else: + up_pointer += 1 + down_pointer = 0 + pointer_list = [] + + + start = pointer_list[0] + end = pointer_list[-1] + bert_text_list = list(bert_text) + bert_text_list.insert(start, "") + bert_text_list.insert(end + 2 , "") + + text_original_list = list(text_original) + + up = 0 + down = 0 + + while True: + if up == len(text_original_list): + break + + if text_original_list[up] == bert_text_list[down]: + up += 1 + down += 1 + + else: + if bert_text_list[down] == "": + down += 1 + elif bert_text_list[down] == "": + down += 1 + else: + bert_text_list.insert(down, text_original_list[up]) + up += 1 + down += 1 + + bert_text = "".join(bert_text_list) + return True, bert_text + + +def biaohong_bert_predict(sentence_0_list, sentence_1_list): + ''' + 找出标红字符 + :param bool_check_sentense: + :return: + ''' + + # sentence_0_list = [] + # sentence_1_list = [] + # sim_paper_name = [] + # + # for i in biaohong_list: + # sentence_0_list.append("。".join([paper_list[i[0][0]], paper_list[i[0][1]], paper_list[i[0][2]]])) + # sentence_1_list.append("。".join([recall_data_list[i[1][1]], recall_data_list[i[1][1]], recall_data_list[i[1][2]]])) + + paper_dict = dialog_line_parse("http://192.168.31.74:16003/", {"sentence_0": sentence_0_list, "sentence_1": sentence_1_list})["resilt"] + + # paper_dict + # print("原文：".format(i), paper_dict[i][0]) + # print("原文标红：".format(i), paper_dict[i][1]) + # print("相似：".format(i), paper_dict[i][2]) + # print("相似标红：".format(i), paper_dict[i][3]) + + # original_text + # + # + # for paper_dict_dan, sentence_0_dan, sentence_1_dan in zip(paper_dict, sentence_0_list, sentence_1_list): + # original_text_marked_red + + return paper_dict + +def ulit_text(title, text): + data = [] + try: + text = json.loads(text)["content"] + except: + pass + + text = text.strip().replace("\n", "").replace(" ", "").replace("。", "。\n") + text_list = text.split("\n") + + for i in text_list: + data.append([i, title]) + return data + +def run_query(conn, sql, params): + with conn.cursor() as cursor: + cursor.execute(sql, params) + result = cursor.fetchall() + return result + + +def processing_one_text(paper_id): + conn = pymysql.connect( + host='192.168.31.145', + port=3306, + user='root', + password='123456', + db='zhiwang_db', + charset='utf8mb4', + cursorclass=pymysql.cursors.DictCursor + ) + + sql = 'SELECT * FROM main_table_paper_detail_message WHERE doc_id=%s' + params = (paper_id,) + + result = run_query(conn, sql, params) + + conn.close() + print(result[0]['title'], result[0]['author']) + title = result[0]['title'] + author = result[0]['author'] + degree = result[0]['degree'] + year = result[0]['content'].split("/")[5] + content_path = result[0]['content'] + + try: + with open(content_path, encoding="utf-8") as f: + text = f.read() + except: + with open(content_path, encoding="gbk") as f: + text = f.read() + + paper_info = { + "title": title, + "author": author, + "degree": degree, + "year": year, + "paper_len_word": len(text) + } + data = ulit_text(paper_info, text) + return data + + +def ulit_recall_paper(recall_data_list_dict): + ''' + 对返回的十篇文章路径读取并解析 + :param recall_data_list_path: + :return data: list [[sentence, filename],[sentence, filename],[sentence, filename]] + ''' + + # data = [] + # for path in recall_data_list_path: + # filename = path.split("/")[-1] + # with open(path, encoding="gbk") as f: + # text = f.read() + # text_list = text.split("\n") + # for sentence in text_list: + # if sentence != "": + # data.append([sentence, filename]) + # return data + + + data = [] + for i in list(recall_data_list_dict.items())[:5]: + data_one = processing_one_text(i[0]) + data.extend(data_one) + + return data + + +def recall_10(title, abst_zh, content) -> dict: + ''' + 宇鹏召回接口 + :param paper_name: + :return: + ''' + + request_json = { + "title": title, + "abst_zh": abst_zh, + "content": content + } + paper_dict = dialog_line_parse("http://192.168.31.145:50004/check", request_json) + + return paper_dict + + +def uilt_content(content): + zhaiyao_list = ["摘要"] + zhaiyao_en_list = ["Abstract", "abstract"] + mulu_list = ["目录"] + key_word_list = ["关键词"] + key_word_bool = False + key_word_str = "" + zhaiyao_bool = False + zhaiyao_en_bool = False + zhaiyao_str = "" + zhaiyao_en_str = "" + mulu_str = "" + zhaiyao_text = "" + mulu_bool = False + + for i in zhaiyao_list: + if i in content: + zhaiyao_bool = True + zhaiyao_str = i + break + + for i in zhaiyao_en_list: + if i in content: + zhaiyao_en_bool = True + zhaiyao_en_str = i + break + + for i in mulu_list: + if i in content: + mulu_str = i + mulu_bool = True + break + + for i in key_word_list: + if i in content: + key_word_str = i + key_word_bool = True + break + + if zhaiyao_bool== True and zhaiyao_en_bool == True: + pantten_zhaiyao = "{}(.*?){}".format(zhaiyao_str,zhaiyao_en_str) + result_biaoti_list = re.findall(pantten_zhaiyao, content) + zhaiyao_text = result_biaoti_list[0] + + elif zhaiyao_bool == True and key_word_bool == True: + pantten_zhaiyao = "{}(.*?){}".format(zhaiyao_str,key_word_str ) + result_biaoti_list = re.findall(pantten_zhaiyao, content) + zhaiyao_text = result_biaoti_list[0] + + elif zhaiyao_bool == True and mulu_bool == True: + pantten_zhaiyao = "{}(.*?){}".format(zhaiyao_str,mulu_str) + result_biaoti_list = re.findall(pantten_zhaiyao, content) + zhaiyao_text = result_biaoti_list[0] + + return zhaiyao_text + + +def ulit_request_file(file): + file_name = file.filename + if file_name.split(".")[-1] == "txt": + file_name_save = "data/request/{}".format(file_name) + file.save(file_name_save) + try: + with open(file_name_save, encoding="gbk") as f: + content = f.read() + except: + with open(file_name_save, encoding="utf-8") as f: + content = f.read() + + content = content.strip().replace("\n", "").replace(" ", "") + abst_zh = uilt_content(content) + + return abst_zh, content + + + +# @app.route("/", methods=["POST"]) +# def handle_query(): +# print(request.remote_addr) +# +# # request.form.get('prompt') +# dataBases = request.form.get("dataBases") +# minSimilarity = request.form.get("minSimilarity") # txt +# minWords = request.form.get("minWords") +# title = request.form.get("title") +# author = request.form.get("author") # txt +# file = request.files.get('file') +# token = request.form.get("token") +# account = request.form.get("account") +# goodsId = request.form.get("goodsId") +# callbackUrl = request.form.get("callbackUrl") +# +# +# t0 = time.time() +# abst_zh, content = ulit_request_file(file) +# +# # 调用宇鹏查询相似十篇 +# # recall_data_list_dict = recall_10(title, abst_zh, content) +# +# t1 = time.time() +# print("查找相似的50篇完成") +# with open("data/rell_json.txt") as f: +# recall_data_list_dict = eval(f.read()) +# +# # 读取文章转化成格式数据 +# recall_data_list = ulit_recall_paper(recall_data_list_dict) +# print("文章格式转化完成") +# +# # recall_data_list = pd.read_csv("data/10235513_大型商业建筑人员疏散设计研究_沈福禹/查重.csv", encoding="utf-8").values.tolist() +# +# # 进入精确查重系统 +# print("进入精确查重系统") +# return_list = accurate_check_rouge(title, author, content, recall_data_list) +# +# print("召回50篇", t1 - t0) +# +# return_text = {"resilt": return_list, "probabilities": None, "status_code": 200} +# return jsonify(return_text) # 返回结果 + + +def classify(): # 调用模型，设置最大batch_size + while True: + if redis_.llen(db_key_query) == 0: # 若队列中没有元素就继续获取 + time.sleep(3) + continue + query = redis_.lpop(db_key_query).decode('UTF-8') # 获取query的text + data_dict_path = json.loads(query) + path = data_dict_path['path'] + # text_type = data_dict["text_type"] + + with open(path, encoding='utf8') as f1: + # 加载文件的对象 + data_dict = json.load(f1) + + query_id = data_dict['id'] + print(query_id) + dataBases = data_dict['dataBases'] + minSimilarity = data_dict['minSimilarity'] + minWords = data_dict['minWords'] + title = data_dict['title'] + author = data_dict['author'] + abst_zh = data_dict['abst_zh'] + content = data_dict['content'] + token = data_dict['token'] + account = data_dict['account'] + goodsId = data_dict['goodsId'] + callbackUrl = data_dict['callbackUrl'] + + + # 调用宇鹏查询相似十篇 + # recall_data_list_dict = recall_10(title, abst_zh, content) + + t1 = time.time() + print("查找相似的50篇完成") + with open("data/rell_json.txt") as f: + recall_data_list_dict = eval(f.read()) + + # 读取文章转化成格式数据 + recall_data_list = ulit_recall_paper(recall_data_list_dict) + print("文章格式转化完成") + + # recall_data_list = pd.read_csv("data/10235513_大型商业建筑人员疏散设计研究_沈福禹/查重.csv", encoding="utf-8").values.tolist() + + # 进入精确查重系统 + print("进入精确查重系统") + return_list = accurate_check_rouge(title, author, content, recall_data_list) + + return_text = {"resilt": return_list, "probabilities": None, "status_code": 200} + + load_result_path = "./new_data_logs/{}.json".format(query_id) + + print("query_id: ", query_id) + print("load_result_path: ", load_result_path) + + with open(load_result_path, 'w', encoding='utf8') as f2: + # ensure_ascii=False才能输入中文，否则是Unicode字符 + # indent=2 JSON数据的缩进，美观 + json.dump(return_text, f2, ensure_ascii=False, indent=4) + + print(query_id) + print(load_result_path) + redis_.set(query_id, load_result_path, 86400) + redis_.srem(db_key_querying, query_id) + + +@app.route("/", methods=["POST"]) +def handle_query(): + try: + print(request.remote_addr) + + # request.form.get('prompt') + dataBases = request.form.get("dataBases") + minSimilarity = request.form.get("minSimilarity") # txt + minWords = request.form.get("minWords") + title = request.form.get("title") + author = request.form.get("author") # txt + file = request.files.get('file') + token = request.form.get("token") + account = request.form.get("account") + goodsId = request.form.get("goodsId") + callbackUrl = request.form.get("callbackUrl") + + abst_zh, content = ulit_request_file(file) + + id_ = str(uuid.uuid1()) # 为query生成唯一标识 + print("uuid: ", uuid) + print(id_) + d = { + 'id': id_, + 'dataBases': dataBases, + 'minSimilarity': minSimilarity, + 'minWords': minWords, + 'title': title, + 'author': author, + 'abst_zh': abst_zh, + 'content': content, + 'token': token, + 'account': account, + 'goodsId': goodsId, + 'callbackUrl': callbackUrl + } + + # 绑定文本和query id + print(d) + load_request_path = './request_data_logs/{}.json'.format(id_) + with open(load_request_path, 'w', encoding='utf8') as f2: + # ensure_ascii=False才能输入中文，否则是Unicode字符 + # indent=2 JSON数据的缩进，美观 + json.dump(d, f2, ensure_ascii=False, indent=4) + redis_.rpush(db_key_query, json.dumps({"id": id_, "path": load_request_path})) # 加入redis + redis_.sadd(db_key_querying, id_) + redis_.sadd(db_key_queryset, id_) + return_text = { + 'code': 0, + 'msg': "请求成功", + 'data': { + 'balances': "", + 'orderId': id_, + 'consumeNum': "" + } + } + + print("ok") + except: + return_text = {'code': 1} + return jsonify(return_text) # 返回结果 + +t = Thread(target=classify) +t.start() + +if __name__ == "__main__": + app.run(host="0.0.0.0", port=16001, threaded=True, debug=True) \ No newline at end of file