duplicate_check/flask_check_bert_test.py

import os
import numpy as np
from numpy.linalg import norm
import pandas as pd
# from rouge import Rouge
from rouge_chinese import Rouge
from Rouge_w import Rouge_w,Rouge_l
import json
import pymysql
import re
import requests
from flask import Flask, jsonify
from flask import request
import uuid
import time
import redis
from threading import Thread
from multiprocessing import Pool
app = Flask(__name__)
app.config["JSON_AS_ASCII"] = False

pool = redis.ConnectionPool(host='localhost', port=63179, max_connections=100, db=7, password="zhicheng123*")
redis_ = redis.Redis(connection_pool=pool, decode_responses=True)

db_key_query = 'query'
db_key_querying = 'querying'
db_key_queryset = 'queryset'

nums_cpus = 24
rouge = Rouge()
rouge_model = Rouge_w()
rouge_l_model = Rouge_l()


def bert_check(text, recall_data_list):
    '''
    bert 查重
    :return:
    '''

    sen_0 = [text] * len(recall_data_list)
    sen_1 = [i[0] for i in recall_data_list]

    return_list = []
    request_json = {
        "texts": [sen_0, sen_1],
    }
    paper_dict = dialog_line_parse("http://192.168.31.74:16002/", request_json)
    score_list = paper_dict["res"]

    # 后期要改
    # return_list.append(re1[0][1])
    # return_list.append(re1[0][0])
    if 1 in score_list:
        index_score = score_list.index(1)
    else:
        index_score = "NaN"

    if index_score == "NaN":
        return_list.append(0)
        return_list.append("")
    else:
        return_list.append(1)
        return_list.append(index_score)

    return return_list



def rouge_value_self(data_1, data_2):
    data_1 = [' '.join(i) for i in data_1]
    data_2 = [' '.join(i) for i in data_2]
    rouge_l_list = []

    for sen_1, sen_2 in zip(data_1, data_2):
        sen_1 = sen_1.split(" ")
        sen_2 = sen_2.split(" ")
        rouge_l_score = rouge_l_model.score(sen_1, sen_2)
        rouge_l_list.append(rouge_l_score)

    return "", "", rouge_l_list


def rouge_pre(text, df_train_nuoche):

    return_list = []
    index_rouge_list = []
    text_list = [text] * len(df_train_nuoche)

    data_list = []
    for data_dan in df_train_nuoche:
        data_list.append(data_dan[0])
    rouge_1, rouge_2, rouge_l = rouge_value_self(text_list, data_list)
    index_rouge_list.extend(rouge_l)

    re1 = [(i[0], i[1]) for i in sorted(list(enumerate(index_rouge_list)), key=lambda x: x[1], reverse=True)]

    return_list.append(re1[0][1])
    return_list.append(re1[0][0])

    return return_list


def rouge_pre_m(text, df_train_nuoche):

    return_list = []
    index_rouge_list = []

    text_list = [text] * len(df_train_nuoche)

    data_list = []
    for data_dan in df_train_nuoche:
        data_list.append(data_dan[0])
    rouge_1, rouge_2, rouge_l = rouge_value_self(text_list, data_list)
    index_rouge_list.extend(rouge_l)

    re1 = [(i[0], i[1]) for i in sorted(list(enumerate(index_rouge_list)), key=lambda x: x[1], reverse=True)]

    return_list.append(re1[0][1])
    return_list.append(re1[0][0])

    return return_list


def accurate_check_rouge(
        title,
        author,
        text_paper,
        recall_data_list
    ):
    '''
    精确查重出相似句子
    :param text:
    :param recall_data_list: list [[sentence, filename],[sentence, filename],[sentence, filename]]
    :return:
    '''
    # 文本处理
    centent_list = []
    text_paper = str(text_paper).replace("。\n", "。")
    centent_list.extend(text_paper.split("。"))
    data_zong = []
    sentence_word_nums = 0

    # rouge算法查重
    rst = []
    p = Pool(nums_cpus)  # 进程池中含有n个子进程

    print("centent_list", centent_list)

    for i in range(len(centent_list)):
        text = centent_list[i]
        a = p.apply_async(rouge_pre_m, args=(text, recall_data_list,))
        rst.append(a)
    p.close()
    p.join()  # 等待所有子进程执行完毕。调用join()之前必须先调用close()，调用close()之后就不能继续添加新的Process了。

    rst = [i.get() for i in rst]

    for i in range(len(rst)):
        print(rst[i])
        data_zong.append(rst[i])

    t0 = time.time()
    # bert算法查重
    # for text in centent_list:
    #     bert_pre_list = bert_check(text, recall_data_list)
    #     data_zong.append(bert_pre_list)
    t1 = time.time()
    original_dict = []


    # 找出相似的句子序号
    bool_check_sentense = []
    # bert算法
    # for i in range(len(data_zong)):
    #     if data_zong[i][0] == 1:
    #         bool_check_sentense.append([i,data_zong[i][1]])

    # rouge算法
    for i in range(len(data_zong)):
        if data_zong[i][0] > 0.47:
            bool_check_sentense.append([i,data_zong[i][1]])
    biao_red = biaohong(bool_check_sentense, data_zong, recall_data_list) # [[[0, 1, 2], [479, 480, 481]], [[3, 4, 5], [481, 482, 483]], [[6, 7, 8], [484, 485, 486]]]

    print("bert精确查重时间", t1-t0)


    sentence_0_list = []
    sentence_1_list = []
    sim_paper_name = []

    for i in biao_red:
        if recall_data_list[i[1][0]][1] == recall_data_list[i[1][1]][1] == recall_data_list[i[1][2]][1]:
            sentence_0_list.append("。".join([centent_list[i[0][0]], centent_list[i[0][1]], centent_list[i[0][2]]]))
            sentence_1_list.append("".join([recall_data_list[i[1][0]][0], recall_data_list[i[1][1]][0], recall_data_list[i[1][2]][0]]))
            sim_paper_name.append(recall_data_list[i[1][0]][1])
        else:
            continue

    sentence_0_list_new = []
    sentence_1_list_new = []


    for i in zip(sentence_0_list, sentence_1_list):
        if len(i[0]) + len(i[1]) < 1200:
            sentence_0_list_new.append(i[0])
            sentence_1_list_new.append(i[1])
        else:
            print(len(i[0]) + len(i[1]))
            continue
    t2 = time.time()
    paper_dict = biaohong_bert_predict(sentence_0_list_new, sentence_1_list_new)

    t3 = time.time()
    print("标红时间", t3 - t2)
    original_text = []
    original_text_contrast = []
    repeat_quote_info = []

    chongfuwendang = {}

    for paper_dict_dan_id, sentence_0_dan, sentence_1_dan, sim_paper_name_dan in zip(range(len(paper_dict)), sentence_0_list_new, sentence_1_list_new, sim_paper_name):

        print([sentence_0_dan, sentence_1_dan])
        original_text_contrast_dict = {
            "original_text": "",
            "similar_content": [
                {
                    "content": "",
                    "thesis_info": "",
                    "title": "",
                    "year": "",
                    "degree": "",
                    "author": "",
                }
            ]
        }
        try:
            sentence_0_bool, sentence_0_dan_red = original_text_marked_red(sentence_0_dan, paper_dict[paper_dict_dan_id][0], paper_dict[paper_dict_dan_id][1]) # text_original, bert_text, bert_text_pre
        except:
            print("报错", [sentence_0_dan, paper_dict[paper_dict_dan_id][0], paper_dict[paper_dict_dan_id][1]])
            continue
            # 9/0
        sentence_1_bool, sentence_1_dan_red = original_text_marked_red(sentence_1_dan, paper_dict[paper_dict_dan_id][2], paper_dict[paper_dict_dan_id][3]) # text_original, bert_text, bert_text_pre

        if sentence_0_bool == False or sentence_1_bool == False:
            continue

        dan_sentence_word_nums = len(paper_dict[paper_dict_dan_id][1])
        sentence_word_nums += dan_sentence_word_nums

        original_text.append(sentence_0_dan_red)
        original_text_contrast_dict["original_text"] = "此处有 {} 字相似\n".format(
            dan_sentence_word_nums) + sentence_0_dan_red

        thesis_info = " ".join([sim_paper_name_dan["title"], sim_paper_name_dan["author"], sim_paper_name_dan["degree"], sim_paper_name_dan["year"]])
        original_text_contrast_dict["similar_content"][0]["content"] = sentence_1_dan_red
        original_text_contrast_dict["similar_content"][0]["title"] = sim_paper_name_dan["title"]
        original_text_contrast_dict["similar_content"][0]["author"] = sim_paper_name_dan["author"]
        original_text_contrast_dict["similar_content"][0]["degree"] = sim_paper_name_dan["degree"]
        original_text_contrast_dict["similar_content"][0]["year"] = sim_paper_name_dan["year"]
        original_text_contrast_dict["similar_content"][0]["thesis_info"] = thesis_info

        original_text_contrast.append(original_text_contrast_dict)

        # for i in repeat_quote_info:
        #     if

        if thesis_info not in chongfuwendang:
            chongfuwendang[thesis_info] = {
                "quote": False,
                "thesis_author": sim_paper_name_dan["author"],
                "thesis_date" : sim_paper_name_dan["year"],
                "thesis_info" : thesis_info,
                "thesis_repeat_rate": (dan_sentence_word_nums/sim_paper_name_dan["paper_len_word"]) * 100, #round(repetition_rate, 3) * 100
                "thesis_title": sim_paper_name_dan["title"],
                "thesis_link": "",
                "thesis_publish": sim_paper_name_dan["degree"],
                "thesis_repeat_word": dan_sentence_word_nums,
                "thesis_teacher": "",
                "paper_len_word": sim_paper_name_dan["paper_len_word"]
            }
        else:
            chongfuwendang[thesis_info]["thesis_repeat_word"] += dan_sentence_word_nums
            chongfuwendang[thesis_info]["thesis_repeat_rate"] = (chongfuwendang[thesis_info]["thesis_repeat_word"]/chongfuwendang[thesis_info]["paper_len_word"]) * 100


    chongfuwendang = sorted(chongfuwendang.items(),
                            key=lambda x: x[1]["thesis_repeat_rate"], reverse=False)


    for i in range(len(chongfuwendang)):
        repeat_paper_one_info_dict = chongfuwendang[i][1]
        repeat_paper_one_info_dict.pop("paper_len_word")
        repeat_paper_one_info_dict["thesis_repeat_rate"] = str(round(repeat_paper_one_info_dict["thesis_repeat_rate"], 1)) + "%"
        repeat_quote_info.append(repeat_paper_one_info_dict)

    original_text = "。".join(original_text)

    repetition_rate = sentence_word_nums/len(text_paper)
    repetition_rate = round(repetition_rate, 3) * 100

    format = '%Y-%m-%d %H:%M:%S'
    value = time.localtime(int(time.time()))
    dt = time.strftime(format, value)

    return {
        "author": author,
        "check_time": dt,
        "title": title,
        "time_range": "1900-01-01至2023-08-08",
        "section_data": [
            {
                "oneself_repeat_words": sentence_word_nums,
                "reference_repeat_words": sentence_word_nums,
                "section_name": "第1部分",
                "section_oneself_rate": "{}%".format(repetition_rate),
                "section_repeat_rate": "{}%".format(repetition_rate),
                "section_repeat_words": sentence_word_nums,
                "section_words": len(text_paper)
            }
        ],
        "section_details": [
            {
                "end_page_index": 0,
                "name": "",
                "repeat_rate": "",
                "repeat_words": "",
                "words": "",
                "original_text": original_text,
                "original_text_oneself": original_text,
                "original_text_contrast": original_text_contrast,
                "repeat_quote_info": repeat_quote_info
            }
        ],
        "total_data": {
            "back_repeat_words": "",
            "exclude_personal_rate": "{}%".format(repetition_rate),
            "exclude_quote_rate": "{}%".format(repetition_rate),
            "foot_end_note": "0",
            "front_repeat_words": "",
            "single_max_rate": "",
            "single_max_repeat_words": "",
            "suspected_paragraph": "1",
            "suspected_paragraph_max_repeat_words": "",
            "suspected_paragraph_min_repeat_words": "",
            "tables": "0",
            "total_paragraph": "1",
            "total_repeat_rate": "{}%".format(repetition_rate),
            "total_repeat_words": sentence_word_nums,
            "total_words": len(text_paper)
        }
    }





def biaohong(bool_check_sentense, data_zong, df_train_nuoche):
    '''
    标红的序号 [[0,1,2],[3,4,5]]
    :param bool_check_sentense:
    :return: list
    '''
    biao_red = []
    i = 0
    start = -1
    end = -1
    while True:
        if i >= len(bool_check_sentense) or bool_check_sentense[i][0] +1 >= len(data_zong) or bool_check_sentense[i][1]+1 >= len(df_train_nuoche):
            break
        elif bool_check_sentense[i][0]-1 == start:
            i += 1
            continue
        elif bool_check_sentense[i][0] == end:
            i += 1
            continue
        elif bool_check_sentense[i][0]-1 == end:
            i += 1
            continue
        else:
            biao_red_dan = []
            biao_red_dan.append([bool_check_sentense[i][0] - 1, bool_check_sentense[i][1] - 1])
            biao_red_dan.append([bool_check_sentense[i][0], bool_check_sentense[i][1]])
            biao_red_dan.append([bool_check_sentense[i][0] + 1, bool_check_sentense[i][1] + 1])
            biao_red.append([[bool_check_sentense[i][0]-1, bool_check_sentense[i][0], bool_check_sentense[i][0]+1],
                             [bool_check_sentense[i][1]-1, bool_check_sentense[i][1], bool_check_sentense[i][1]+1]])
            start = bool_check_sentense[i][0]-1
            end = bool_check_sentense[i][0]+1
            i += 1

    return biao_red


def dialog_line_parse(url, text):
    """
    将数据输入模型进行分析并输出结果
    :param url: 模型url
    :param text: 进入模型的数据
    :return: 模型返回结果
    """

    response = requests.post(
        url,
        json=text,
        timeout=100000
    )
    if response.status_code == 200:
        return response.json()
    else:
        # logger.error(
        #     "【{}】 Failed to get a proper response from remote "
        #     "server. Status Code: {}. Response: {}"
        #     "".format(url, response.status_code, response.text)
        # )
        print("【{}】 Failed to get a proper response from remote "
            "server. Status Code: {}. Response: {}"
            "".format(url, response.status_code, response.text))
        print(text)
        return {}


def is_english_char(char):
    code = ord(char)
    return 32 <= code <= 126


def original_text_marked_red(text_original, bert_text, bert_text_pre):
    '''
    把原文标红字段找到
    :param text_original:
    :param bert_text:
    :param bert_text_pre:
    :return:
    '''

    fuhao = ["\n"]
    up_pointer = 0
    down_pointer = 0

    pointer_list = []

    if len(bert_text_pre) > len(bert_text):
        return False, ""

    while True:
        if down_pointer >= len(bert_text_pre):
            break
        elif down_pointer == len(bert_text_pre)-1:
            if bert_text[up_pointer] == bert_text_pre[down_pointer]:
                pointer_list.append(up_pointer)
                break
            else:
                up_pointer += 1
                down_pointer = 0
                pointer_list = []

        elif bert_text[up_pointer] in fuhao:
            up_pointer += 1

        else:
            if bert_text[up_pointer] == bert_text_pre[down_pointer]:
                pointer_list.append(up_pointer)
                up_pointer += 1
                down_pointer += 1
            else:
                if bert_text_pre[down_pointer:down_pointer+5] == "[UNK]":
                    up_pointer += 1
                    down_pointer += 5
                    pointer_list.append(up_pointer)
                elif is_english_char(bert_text_pre[down_pointer]) == True:
                    up_pointer += 1
                    down_pointer += 1
                    pointer_list.append(up_pointer)
                else:
                    up_pointer += 1
                    down_pointer = 0
                    pointer_list = []


    start = pointer_list[0]
    end = pointer_list[-1]
    bert_text_list = list(bert_text)
    bert_text_list.insert(start, "<red>")
    bert_text_list.insert(end + 2 , "</red>")

    text_original_list = list(text_original)

    up = 0
    down = 0

    while True:
        if up == len(text_original_list):
            break

        if text_original_list[up] == bert_text_list[down]:
            up += 1
            down += 1

        else:
            if bert_text_list[down] == "<red>":
                down += 1
            elif bert_text_list[down] == "</red>":
                down += 1
            else:
                bert_text_list.insert(down, text_original_list[up])
                up += 1
                down += 1

    bert_text = "".join(bert_text_list)
    return True, bert_text


def biaohong_bert_predict(sentence_0_list, sentence_1_list):
    '''
    找出标红字符
    :param bool_check_sentense:
    :return:
    '''

    # sentence_0_list = []
    # sentence_1_list = []
    # sim_paper_name = []
    #
    # for i in biaohong_list:
    #     sentence_0_list.append("。".join([paper_list[i[0][0]], paper_list[i[0][1]], paper_list[i[0][2]]]))
    #     sentence_1_list.append("。".join([recall_data_list[i[1][1]], recall_data_list[i[1][1]], recall_data_list[i[1][2]]]))

    paper_dict = dialog_line_parse("http://192.168.31.74:16003/", {"sentence_0": sentence_0_list, "sentence_1": sentence_1_list})["resilt"]

    # paper_dict
    # print("原文：".format(i), paper_dict[i][0])
    # print("原文标红：".format(i), paper_dict[i][1])
    # print("相似：".format(i), paper_dict[i][2])
    # print("相似标红：".format(i), paper_dict[i][3])

    # original_text
    #
    #
    # for paper_dict_dan, sentence_0_dan, sentence_1_dan in zip(paper_dict, sentence_0_list, sentence_1_list):
    #     original_text_marked_red

    return paper_dict

def ulit_text(title, text):
    data = []
    try:
        text = json.loads(text)["content"]
    except:
        pass

    text = text.strip().replace("\n", "").replace(" ", "").replace("。", "。\n")
    text_list = text.split("\n")

    for i in text_list:
        data.append([i, title])
    return data

def run_query(conn, sql, params):
    with conn.cursor() as cursor:
        cursor.execute(sql, params)
        result = cursor.fetchall()
    return result


def processing_one_text(paper_id):
    conn = pymysql.connect(
        host='192.168.31.145',
        port=3306,
        user='root',
        password='123456',
        db='zhiwang_db',
        charset='utf8mb4',
        cursorclass=pymysql.cursors.DictCursor
    )

    sql = 'SELECT * FROM main_table_paper_detail_message WHERE doc_id=%s'
    params = (paper_id,)

    result = run_query(conn, sql, params)

    conn.close()
    print(result[0]['title'], result[0]['author'])
    title = result[0]['title']
    author = result[0]['author']
    degree = result[0]['degree']
    year = result[0]['content'].split("/")[5]
    content_path = result[0]['content']

    try:
        with open(content_path, encoding="utf-8") as f:
            text = f.read()
    except:
        with open(content_path, encoding="gbk") as f:
            text = f.read()

    paper_info = {
        "title": title,
        "author": author,
        "degree": degree,
        "year": year,
        "paper_len_word": len(text)
    }
    data = ulit_text(paper_info, text)
    return data


def ulit_recall_paper(recall_data_list_dict):
    '''
    对返回的十篇文章路径读取并解析
    :param recall_data_list_path:
    :return data: list [[sentence, filename],[sentence, filename],[sentence, filename]]
    '''

    # data = []
    # for path in recall_data_list_path:
    #     filename = path.split("/")[-1]
    #     with open(path, encoding="gbk") as f:
    #         text = f.read()
    #     text_list = text.split("\n")
    #     for sentence in text_list:
    #         if sentence != "":
    #             data.append([sentence, filename])
    # return data


    data = []
    for i in list(recall_data_list_dict.items())[:5]:
        data_one = processing_one_text(i[0])
        data.extend(data_one)

    return data


def recall_10(title, abst_zh, content) -> dict:
    '''
    宇鹏召回接口
    :param paper_name:
    :return:
    '''

    request_json = {
        "title": title,
        "abst_zh": abst_zh,
        "content": content
    }
    paper_dict = dialog_line_parse("http://192.168.31.145:50004/check", request_json)

    return paper_dict


def uilt_content(content):
    zhaiyao_list = ["摘要"]
    zhaiyao_en_list = ["Abstract", "abstract"]
    mulu_list = ["目录"]
    key_word_list = ["关键词"]
    key_word_bool = False
    key_word_str = ""
    zhaiyao_bool = False
    zhaiyao_en_bool = False
    zhaiyao_str = ""
    zhaiyao_en_str = ""
    mulu_str = ""
    zhaiyao_text = ""
    mulu_bool = False

    for i in zhaiyao_list:
        if i in content:
            zhaiyao_bool = True
            zhaiyao_str = i
            break

    for i in zhaiyao_en_list:
        if i in content:
            zhaiyao_en_bool = True
            zhaiyao_en_str = i
            break

    for i in mulu_list:
        if i in content:
            mulu_str = i
            mulu_bool = True
            break

    for i in key_word_list:
        if i in content:
            key_word_str = i
            key_word_bool = True
            break

    if zhaiyao_bool== True and zhaiyao_en_bool == True:
        pantten_zhaiyao = "{}(.*?){}".format(zhaiyao_str,zhaiyao_en_str)
        result_biaoti_list = re.findall(pantten_zhaiyao, content)
        zhaiyao_text = result_biaoti_list[0]

    elif zhaiyao_bool == True and key_word_bool == True:
        pantten_zhaiyao  = "{}(.*?){}".format(zhaiyao_str,key_word_str )
        result_biaoti_list = re.findall(pantten_zhaiyao, content)
        zhaiyao_text = result_biaoti_list[0]

    elif zhaiyao_bool == True and mulu_bool == True:
        pantten_zhaiyao  = "{}(.*?){}".format(zhaiyao_str,mulu_str)
        result_biaoti_list = re.findall(pantten_zhaiyao, content)
        zhaiyao_text = result_biaoti_list[0]

    return zhaiyao_text


def ulit_request_file(file):
    file_name = file.filename
    if file_name.split(".")[-1] == "txt":
        file_name_save = "data/request/{}".format(file_name)
        file.save(file_name_save)
        try:
            with open(file_name_save, encoding="gbk") as f:
                content = f.read()
        except:
            with open(file_name_save, encoding="utf-8") as f:
                content = f.read()

    content = content.strip().replace("\n", "").replace(" ", "")
    abst_zh = uilt_content(content)

    return abst_zh, content



# @app.route("/", methods=["POST"])
# def handle_query():
#     print(request.remote_addr)
#
#     # request.form.get('prompt')
#     dataBases = request.form.get("dataBases")
#     minSimilarity = request.form.get("minSimilarity") # txt
#     minWords = request.form.get("minWords")
#     title = request.form.get("title")
#     author = request.form.get("author") # txt
#     file = request.files.get('file')
#     token = request.form.get("token")
#     account = request.form.get("account")
#     goodsId = request.form.get("goodsId")
#     callbackUrl = request.form.get("callbackUrl")
#
#
#     t0 = time.time()
#     abst_zh, content = ulit_request_file(file)
#
#     # 调用宇鹏查询相似十篇
#     # recall_data_list_dict = recall_10(title, abst_zh, content)
#
#     t1 = time.time()
#     print("查找相似的50篇完成")
#     with open("data/rell_json.txt") as f:
#         recall_data_list_dict = eval(f.read())
#
#     # 读取文章转化成格式数据
#     recall_data_list = ulit_recall_paper(recall_data_list_dict)
#     print("文章格式转化完成")
#
#     # recall_data_list = pd.read_csv("data/10235513_大型商业建筑人员疏散设计研究_沈福禹/查重.csv", encoding="utf-8").values.tolist()
#
#     # 进入精确查重系统
#     print("进入精确查重系统")
#     return_list = accurate_check_rouge(title, author, content, recall_data_list)
#
#     print("召回50篇", t1 - t0)
#
#     return_text = {"resilt": return_list, "probabilities": None, "status_code": 200}
#     return jsonify(return_text)  # 返回结果


def classify():  # 调用模型，设置最大batch_size
    while True:
        if redis_.llen(db_key_query) == 0:  # 若队列中没有元素就继续获取
            time.sleep(3)
            continue
        query = redis_.lpop(db_key_query).decode('UTF-8')  # 获取query的text
        data_dict_path = json.loads(query)
        path = data_dict_path['path']
        # text_type = data_dict["text_type"]

        with open(path, encoding='utf8') as f1:
            # 加载文件的对象
            data_dict = json.load(f1)

        query_id = data_dict['id']
        print(query_id)
        dataBases = data_dict['dataBases']
        minSimilarity = data_dict['minSimilarity']
        minWords = data_dict['minWords']
        title = data_dict['title']
        author = data_dict['author']
        abst_zh = data_dict['abst_zh']
        content = data_dict['content']
        token = data_dict['token']
        account = data_dict['account']
        goodsId = data_dict['goodsId']
        callbackUrl = data_dict['callbackUrl']


        # 调用宇鹏查询相似十篇
        # recall_data_list_dict = recall_10(title, abst_zh, content)

        t1 = time.time()
        print("查找相似的50篇完成")
        with open("data/rell_json.txt") as f:
            recall_data_list_dict = eval(f.read())

        # 读取文章转化成格式数据
        recall_data_list = ulit_recall_paper(recall_data_list_dict)
        print("文章格式转化完成")

        # recall_data_list = pd.read_csv("data/10235513_大型商业建筑人员疏散设计研究_沈福禹/查重.csv", encoding="utf-8").values.tolist()

        # 进入精确查重系统
        print("进入精确查重系统")
        return_list = accurate_check_rouge(title, author, content, recall_data_list)

        return_text = {"resilt": return_list, "probabilities": None, "status_code": 200}

        load_result_path = "./new_data_logs/{}.json".format(query_id)

        print("query_id: ", query_id)
        print("load_result_path: ", load_result_path)

        with open(load_result_path, 'w', encoding='utf8') as f2:
            # ensure_ascii=False才能输入中文，否则是Unicode字符
            # indent=2 JSON数据的缩进，美观
            json.dump(return_text, f2, ensure_ascii=False, indent=4)

        print(query_id)
        print(load_result_path)
        redis_.set(query_id, load_result_path, 86400)
        redis_.srem(db_key_querying, query_id)


@app.route("/", methods=["POST"])
def handle_query():
    try:
        print(request.remote_addr)

        # request.form.get('prompt')
        dataBases = request.form.get("dataBases")
        minSimilarity = request.form.get("minSimilarity") # txt
        minWords = request.form.get("minWords")
        title = request.form.get("title")
        author = request.form.get("author") # txt
        file = request.files.get('file')
        token = request.form.get("token")
        account = request.form.get("account")
        goodsId = request.form.get("goodsId")
        callbackUrl = request.form.get("callbackUrl")

        abst_zh, content = ulit_request_file(file)

        id_ = str(uuid.uuid1())  # 为query生成唯一标识
        print("uuid: ", uuid)
        print(id_)
        d = {
            'id': id_,
            'dataBases': dataBases,
            'minSimilarity': minSimilarity,
            'minWords': minWords,
            'title': title,
            'author': author,
            'abst_zh': abst_zh,
            'content': content,
            'token': token,
            'account': account,
            'goodsId': goodsId,
            'callbackUrl': callbackUrl
            }

        # 绑定文本和query id
        print(d)
        load_request_path = './request_data_logs/{}.json'.format(id_)
        with open(load_request_path, 'w', encoding='utf8') as f2:
            # ensure_ascii=False才能输入中文，否则是Unicode字符
            # indent=2 JSON数据的缩进，美观
            json.dump(d, f2, ensure_ascii=False, indent=4)
        redis_.rpush(db_key_query, json.dumps({"id": id_, "path": load_request_path}))  # 加入redis
        redis_.sadd(db_key_querying, id_)
        redis_.sadd(db_key_queryset, id_)
        return_text = {
            'code': 0,
            'msg': "请求成功",
            'data': {
                'balances': "",
                'orderId': id_,
                'consumeNum': ""
            }
        }

        print("ok")
    except:
        return_text = {'code': 1}
    return jsonify(return_text)  # 返回结果

t = Thread(target=classify)
t.start()

if __name__ == "__main__":
    app.run(host="0.0.0.0", port=16001, threaded=True, debug=True)