From 819f3b9f9f1ef4f3a1faaa4b58179c1971658939 Mon Sep 17 00:00:00 2001 From: "majiahui@haimaqingfan.com" Date: Tue, 11 Apr 2023 10:06:50 +0800 Subject: [PATCH] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E5=A4=9Aip=E5=A4=9A=E7=AF=87?= =?UTF-8?q?=E5=B9=B6=E5=8F=91flask=EF=BC=8C=E8=A7=A3=E5=86=B3api=E6=8E=A5?= =?UTF-8?q?=E5=8F=A3=E6=8A=A5=E9=94=99=E9=97=AE=E9=A2=98=EF=BC=8C=E8=A7=A3?= =?UTF-8?q?=E5=86=B3=E5=A4=9Akey=E5=88=86=E9=85=8D=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- chatGPT账号 26.txt | 5 +- chatgpt_post.py | 2 +- flask_serve_1.py | 803 +++++++++++++++++++++++++++++++++++++++++++++++++++ flask_serve_2.py | 163 ++++++++--- flask_serve_3.py | 724 ++++++++++++++++++++++++++++++++++++++++++++++ flask_sever_1.py | 719 --------------------------------------------- ip对应关系 | 3 + jar_bagk.py | 5 + run_serves.sh | 1 + serve_config_1.py | 31 +- serve_config_2.py | 14 +- serve_config_3.py | 39 +++ 查询uuid.py | 24 +- 测试生成uuid.py | 11 +- 14 files changed, 1755 insertions(+), 789 deletions(-) create mode 100644 flask_serve_1.py create mode 100644 flask_serve_3.py delete mode 100644 flask_sever_1.py create mode 100644 ip对应关系 create mode 100644 run_serves.sh create mode 100644 serve_config_3.py diff --git a/chatGPT账号 26.txt b/chatGPT账号 26.txt index 25a0b20..c771cb5 100644 --- a/chatGPT账号 26.txt +++ b/chatGPT账号 26.txt @@ -8,6 +8,8 @@ romypynysy76433@gmail.com----x87zd8qdq9gg27kw----h4oizvpsykhz0kfn@gmx.com sk-IYY jesikajoya819@gmail.com----cdekq9gq9q9btz7h----vasf5ofm6txmojis@proton.me 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 ounrzagrdezi@gmail.com----tkfevgjsbpcc54w0----qflrsz92ebbkot97@gmx.com api-keys：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 aiyaaiaj27262@gmail.com----m4h48gamzqp48us9----aibmmesoooduwjmo@mail.com api-keys: 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 + + homeworkshxjdhd@gmail.com----fjyysmunpoce16w6----yq29o1u911v0vw5n@outlook.com api-keys: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 hansikamutvani@gmail.com----g794fak5wy9mobqp----vu3gu1ou0nz48u0a@hotmail.com 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 qatalpend356@gmail.com----xo894ftgachr1huq----pg7gnc4a7l1wly1y@proton.me api-keys: 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 @@ -15,13 +17,14 @@ akhiakther8373@gmail.com----dt6g2tp0qkp8t24f----ys1w8k6j0wo6l144@aol.com sk-2bhh qatalpend356@gmail.com----xo894ftgachr1huq----pg7gnc4a7l1wly1y@proton.me api-keys: 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 ramtonazkine@gmail.com----qquh8uaabgdznrtg----exsypsq2buqxwbfa@gmx.com api-keys: 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 davislaird085@gmail.com----cw1fhcde8syffqmy----trre1iri5mhd2v8w@outlook.com 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 + ruiduphfo35644@gmail.com----hbu6458aa44f9s8h----ns237070stg1v5kn@163.com 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 najminmohonas3@gmail.com----mctpqhe06opspbnt----i9m52jmf3i4ngs6a@mail.com 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 eobdkxpwpekd@gmail.com----cpqqxxqk0wf34cba----j59ix0ibxwmlzohf@mail.com api-keys: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 subornasalinia@gmail.com----tn4tved9akum1kx1----ztxqc4kbttzp75om@outlook.com api-keys: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 peswjfgsdsv122@gmail.com----cwsgftzg85h0npx6----baua3t3317j007tx@gmx.com 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 yyzhao@entroduction.cn she220877 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 -JacobRios531@outlook.com----bkwga59Kw7----org-IkKh08PJohhA8ISPAu3F1CoH----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 +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 副号 diff --git a/chatgpt_post.py b/chatgpt_post.py index 4fe7864..56cc2f7 100644 --- a/chatgpt_post.py +++ b/chatgpt_post.py @@ -1,7 +1,7 @@ import requests import json -OPENAI_API_KEY = "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" +OPENAI_API_KEY = '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' url = "https://api.openai.com/v1/chat/completions" headers = { diff --git a/flask_serve_1.py b/flask_serve_1.py new file mode 100644 index 0000000..fbf7920 --- /dev/null +++ b/flask_serve_1.py @@ -0,0 +1,803 @@ +from flask import Flask, jsonify, Response +from flask import request +import redis +import uuid +import json +import time +import threading +from threading import Thread +from flask import send_file, send_from_directory +import os +from flask import make_response +import openai +import base64 +import re +import urllib.parse as pa +import socket +from serve_config_1 import Config +import requests + + +config = Config() + +app = Flask(__name__) +app.config["JSON_AS_ASCII"] = False + +s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) +s.connect(("8.8.8.8", 80)) +localhostip = s.getsockname()[0] + +lock = threading.RLock() +pool = redis.ConnectionPool(host=config.reids_ip, port=config.reids_port, max_connections=50, db=config.reids_db, password=config.reids_password) +redis_ = redis.Redis(connection_pool=pool, decode_responses=True) + +thanks = "致谢" +references = "参考文献" + +flask_serves_env = "http://{}:{}".format(localhostip,config.flask_port) + +paper_download_url = flask_serves_env + "/download?filename_path={}/paper.docx" +paper_start_download_url = flask_serves_env + "/download?filename_path={}/paper_start.docx" + +redis_key_name_openaikey_bad_dict = "openaikey_bad_list_{}".format(str(localhostip)) + +redis_key_name_openaikey_list = "openaikey_list_{}".format(str(localhostip)) + +redis_title = "redis_title" + +redis_title_ing = "redis_title_ing" + +redis_small_task = "redis_small_task" + +redis_res = "redis_res" + +for i in config.openaikey_list: + redis_.rpush(redis_key_name_openaikey_list, i) + +redis_.hset(redis_key_name_openaikey_bad_dict, "1", "1") +redis_.persist(redis_key_name_openaikey_list) +redis_.persist(redis_key_name_openaikey_bad_dict) + + +def request_api_chatgpt(api_key, prompt): + OPENAI_API_KEY = api_key + url = "https://api.openai.com/v1/chat/completions" + headers = { + "Content-Type": "application/json", + "Authorization": f"Bearer {OPENAI_API_KEY}" + } + data = { + "model": "gpt-3.5-turbo", + "messages": [ + {"role": "user", "content": prompt}, + ], + "temperature": 0.5 + } + response = requests.post(url, + headers=headers, + data=json.dumps(data), + timeout=240) + + return response + +def chat_kaitibaogao(api_key, uuid, main_parameter,task_type): + + try: + response =request_api_chatgpt(api_key, config.kaitibaogao_prompt.format(main_parameter[0])) + res = response.json() + kaitibaogao = res["choices"][0]["message"]["content"] + redis_.rpush(redis_key_name_openaikey_list, api_key) + redis_.persist(redis_key_name_openaikey_list) + + except: + """ + 发送警报 + """ + kaitibaogao = "" + + kaitibaogao_task = { + "task_type": task_type, + "uuid": uuid, + "main_parameter": main_parameter + } + time.sleep(3) + small_task = json.dumps(kaitibaogao_task, ensure_ascii=False) + redis_.lpush(redis_small_task, small_task) + + redis_.rpush(redis_key_name_openaikey_list, api_key) + redis_.persist(redis_key_name_openaikey_list) + + redis_.hset(redis_key_name_openaikey_bad_dict, uuid, str((api_key,task_type))) + return + + lock.acquire() + res_dict_str = redis_.hget(redis_res, uuid) + res_dict = json.loads(res_dict_str) + res_dict["tasking_num"] += 1 + print("子任务进度".format(uuid),res_dict["tasking_num"]) + res_dict["开题报告"] = kaitibaogao + res_dict_str = json.dumps(res_dict, ensure_ascii=False) + redis_.hset(redis_res, uuid, res_dict_str) + lock.release() + + +def chat_abstract_keyword(api_key, uuid, main_parameter, task_type): + + try: + # 生成中文摘要 + + response =request_api_chatgpt(api_key, config.chinese_abstract_prompt.format(main_parameter[0])) + res = response.json() + chinese_abstract = res["choices"][0]["message"]["content"] + + # 生成英文的摘要 + + response = request_api_chatgpt(api_key, config.english_abstract_prompt.format(chinese_abstract)) + res = response.json() + english_abstract = res["choices"][0]["message"]["content"] + + + # 生成中文关键字 + + response = request_api_chatgpt(api_key, config.chinese_keyword_prompt.format(chinese_abstract)) + res = response.json() + chinese_keyword = res["choices"][0]["message"]["content"] + + + # 生成英文关键字 + response = request_api_chatgpt(api_key, config.english_keyword_prompt.format(chinese_keyword)) + res = response.json() + english_keyword = res["choices"][0]["message"]["content"] + + redis_.rpush(redis_key_name_openaikey_list, api_key) + redis_.persist(redis_key_name_openaikey_list) + except: + """ + 发送警报 + """ + chinese_abstract = "" + english_abstract = "" + chinese_keyword = "" + english_keyword = "" + + chat_abstract_task = { + "task_type": task_type, + "uuid": uuid, + "main_parameter": main_parameter + } + time.sleep(3) + small_task = json.dumps(chat_abstract_task, ensure_ascii=False) + redis_.lpush(redis_small_task, small_task) + + redis_.rpush(redis_key_name_openaikey_list, api_key) + redis_.persist(redis_key_name_openaikey_list) + + redis_.hset(redis_key_name_openaikey_bad_dict, uuid, str((api_key,task_type))) + return + + paper_abstract_keyword = { + "中文摘要": chinese_abstract, + "英文摘要": english_abstract, + "中文关键词": chinese_keyword, + "英文关键词": english_keyword + } + + # json_str = json.dumps(paper_abstract_keyword, indent=4, ensure_ascii=False) + # abstract_keyword_path = os.path.join(uuid_path, "abstract_keyword.json") + # with open(abstract_keyword_path, 'w') as json_file: + # json_file.write(json_str) + # + # lock.acquire() + # api_key_list.append(api_key) + # lock.release() + + lock.acquire() + res_dict_str = redis_.hget(redis_res, uuid) + res_dict = json.loads(res_dict_str) + res_dict["tasking_num"] += 1 + print("子任务进度".format(uuid),res_dict["tasking_num"]) + res_dict["中文摘要"] = paper_abstract_keyword["中文摘要"] + res_dict["英文摘要"] = paper_abstract_keyword["英文摘要"] + res_dict["中文关键词"] = paper_abstract_keyword["中文关键词"] + res_dict["英文关键词"] = paper_abstract_keyword["英文关键词"] + res_dict_str = json.dumps(res_dict, ensure_ascii=False) + redis_.hset(redis_res, uuid, res_dict_str) + lock.release() + + +def chat_content(api_key, uuid, main_parameter, task_type): + ''' + + :param api_key: + :param uuid: + :param main_parameter: + :return: + ''' + content_index = main_parameter[0] + title = main_parameter[1] + mulu = main_parameter[2] + subtitle = main_parameter[3] + prompt = main_parameter[4] + + if subtitle[:2] == "@@": + res_content = subtitle[2:] + redis_.rpush(redis_key_name_openaikey_list, api_key) + redis_.persist(redis_key_name_openaikey_list) + else: + try: + + response = request_api_chatgpt(api_key, prompt.format(title, mulu, subtitle)) + res = response.json() + res_content = res["choices"][0]["message"]["content"] + + redis_.rpush(redis_key_name_openaikey_list, api_key) + redis_.persist(redis_key_name_openaikey_list) + except: + """ + 发送警报 + """ + res_content = "" + paper_content = { + "task_type": "paper_content", + "uuid": uuid, + "main_parameter": main_parameter + } + time.sleep(3) + small_task = json.dumps(paper_content, ensure_ascii=False) + redis_.lpush(redis_small_task, small_task) + + redis_.rpush(redis_key_name_openaikey_list, api_key) + redis_.persist(redis_key_name_openaikey_list) + + redis_.hset(redis_key_name_openaikey_bad_dict, uuid, str((api_key, task_type))) + return + + lock.acquire() + res_dict_str = redis_.hget(redis_res, uuid) + res_dict = json.loads(res_dict_str) + res_dict["tasking_num"] += 1 + print("子任务进度".format(uuid), res_dict["tasking_num"]) + table_of_contents = res_dict["table_of_contents"] + table_of_contents[content_index] = res_content + res_dict["table_of_contents"] = table_of_contents + res_dict_str = json.dumps(res_dict, ensure_ascii=False) + redis_.hset(redis_res, uuid, res_dict_str) + lock.release() + + +def chat_thanks(api_key, uuid, main_parameter, task_type): + ''' + + :param api_key: + :param uuid: + :param main_parameter: + :return: + ''' + # title, + # thank_prompt + title = main_parameter[0] + prompt = main_parameter[1] + + try: + response = request_api_chatgpt(api_key, prompt.format(title)) + res = response.json() + res_content = res["choices"][0]["message"]["content"] + + redis_.rpush(redis_key_name_openaikey_list, api_key) + redis_.persist(redis_key_name_openaikey_list) + except: + """ + 发送警报 + """ + res_content = "" + thanks_task = { + "task_type": "thanks_task", + "uuid": uuid, + "main_parameter": main_parameter + } + time.sleep(3) + small_task = json.dumps(thanks_task, ensure_ascii=False) + redis_.lpush(redis_small_task, small_task) + + redis_.rpush(redis_key_name_openaikey_list, api_key) + redis_.persist(redis_key_name_openaikey_list) + + redis_.hset(redis_key_name_openaikey_bad_dict, uuid, str((api_key, task_type))) + return + + lock.acquire() + res_dict_str = redis_.hget(redis_res, uuid) + res_dict = json.loads(res_dict_str) + res_dict["tasking_num"] += 1 + print("子任务进度".format(uuid), res_dict["tasking_num"]) + res_dict["致谢"] = res_content + res_dict_str = json.dumps(res_dict, ensure_ascii=False) + redis_.hset(redis_res, uuid, res_dict_str) + lock.release() + + +def chat_references(api_key, uuid, main_parameter, task_type): + ''' + + :param api_key: + :param uuid: + :param main_parameter: + :return: + ''' + # title, + # mulu, + # references_prompt + title = main_parameter[0] + mulu = main_parameter[1] + prompt = main_parameter[2] + try: + + response = request_api_chatgpt(api_key, prompt.format(title, mulu)) + res = response.json() + res_content = res["choices"][0]["message"]["content"] + + redis_.rpush(redis_key_name_openaikey_list, api_key) + redis_.persist(redis_key_name_openaikey_list) + except: + """ + 发送警报 + """ + res_content = "" + + references_task = { + "task_type": "references_task", + "uuid": uuid, + "main_parameter": [ + title, + mulu, + config.references_prompt + ] + } + time.sleep(3) + small_task = json.dumps(references_task, ensure_ascii=False) + redis_.lpush(redis_small_task, small_task) + + redis_.rpush(redis_key_name_openaikey_list, api_key) + redis_.persist(redis_key_name_openaikey_list) + + redis_.hset(redis_key_name_openaikey_bad_dict, uuid, str((api_key, task_type))) + return + + # 加锁 读取resis并存储结果 + lock.acquire() + res_dict_str = redis_.hget(redis_res, uuid) + res_dict = json.loads(res_dict_str) + res_dict["tasking_num"] += 1 + print("子任务进度".format(uuid), res_dict["tasking_num"]) + res_dict["参考文献"] = res_content + res_dict_str = json.dumps(res_dict, ensure_ascii=False) + redis_.hset(redis_res, uuid, res_dict_str) + lock.release() + + +def threading_mulu(api_key, title, uuid): + ''' + 生成目录并吧任务拆解进入子任务的redis_list中和储存结果的redis_list中 + :return: + ''' + try: + response = request_api_chatgpt(api_key, config.mulu_prompt.format(title)) + res = response.json() + mulu = res["choices"][0]["message"]["content"] + + redis_.rpush(redis_key_name_openaikey_list, api_key) + redis_.persist(redis_key_name_openaikey_list) + except: + """ + 发送警报 + """ + res_content = "" + time.sleep(3) + redis_.lpush(redis_title, json.dumps({"id": uuid, "title": title}, ensure_ascii=False)) # 加入redis + redis_.persist(redis_key_name_openaikey_list) + + redis_.rpush(redis_key_name_openaikey_list, api_key) + redis_.persist(redis_key_name_openaikey_list) + + redis_.hset(redis_key_name_openaikey_bad_dict, uuid, str(api_key,"mulu")) + return + + try: + mulu_list = str(mulu).split("\n") + mulu_list = [i.strip() for i in mulu_list if i != ""] + + print(mulu_list) + + cun_bool = False + table_of_contents = [mulu_list[0]] + + for i in mulu_list[1:]: + result_second_biaoti_list = re.findall(config.pantten_second_biaoti, i) + result_other_biaoti_list = re.findall(config.pantten_other_biaoti, i) + if result_second_biaoti_list != []: + table_of_contents.append("@@" + i) + cun_bool = True + continue + if cun_bool == False: + continue + else: + if result_other_biaoti_list != []: + table_of_contents.append("@@" + i) + else: + table_of_contents.append(i) + + print(table_of_contents) + # table_of_contents = table_of_contents[:3] + table_of_contents[-1:] + # print(table_of_contents) + + thanks_references_bool_table = table_of_contents[-3:] + + # thanks = "致谢" + # references = "参考文献" + if references in thanks_references_bool_table: + table_of_contents.remove(references) + + if thanks in thanks_references_bool_table: + table_of_contents.remove(thanks) + + # table_of_contents.append(thanks) + # table_of_contents.append(references) + + # if thanks not in thanks_bool_table: + # table_of_contents.insert(-1, "致谢") + # + # if thanks not in thanks_bool_table: + # table_of_contents.insert(-1, "致谢") + + print(len(table_of_contents)) + + small_task_list = [] + # api_key, + # index, + # title, + # mulu, + # subtitle, + # prompt + kaitibaogao_task = { + "task_type": "kaitibaogao", + "uuid": uuid, + "main_parameter": [title] + } + + chat_abstract_task = { + "task_type": "chat_abstract", + "uuid": uuid, + "main_parameter": [title] + } + small_task_list.append(kaitibaogao_task) + small_task_list.append(chat_abstract_task) + content_index = 0 + while True: + if content_index == len(table_of_contents): + break + subtitle = table_of_contents[content_index] + if content_index == 0: + prompt = config.first_title_prompt + elif subtitle == "参考文献": + prompt = config.references_prompt + elif subtitle == "致谢": + prompt = config.thank_prompt + else: + prompt = config.small_title_prompt + print("请求的所有参数", + content_index, + title, + subtitle, + prompt) + + paper_content = { + "task_type": "paper_content", + "uuid": uuid, + "main_parameter": [ + content_index, + title, + mulu, + subtitle, + prompt + ] + } + + small_task_list.append(paper_content) + content_index += 1 + + thanks_task = { + "task_type": "thanks_task", + "uuid": uuid, + "main_parameter": [ + title, + config.thank_prompt + ] + } + + references_task = { + "task_type": "references_task", + "uuid": uuid, + "main_parameter": [ + title, + mulu, + config.references_prompt + ] + } + + small_task_list.append(thanks_task) + small_task_list.append(references_task) + + for small_task in small_task_list: + small_task = json.dumps(small_task, ensure_ascii=False) + redis_.rpush(redis_small_task, small_task) + + + res = { + "uuid": uuid, + "num_small_task": len(small_task_list), + "tasking_num": 0, + "标题": title, + "目录": mulu, + "开题报告": "", + "任务书": "", + "中文摘要": "", + "英文摘要": "", + "中文关键词": "", + "英文关键词": "", + "正文": "", + "致谢": "", + "参考文献": "", + "table_of_contents": [""] * len(table_of_contents) + } + + res = json.dumps(res, ensure_ascii=False) + redis_.hset(redis_res, uuid, res) + except: + print("目录程序错误") + + +def threading_1(): + # title, redis_key_name_openaikey_list + ''' + 生成目录 + :param title: + :param redis_key_name_openaikey_list: + :return: + ''' + while True: + if redis_.llen(redis_small_task) != 0: # 若队列中有元素就跳过 + time.sleep(1) + continue + elif redis_.llen(redis_title) != 0 and redis_.llen(redis_key_name_openaikey_list) != 0: + title_uuid_dict_str = redis_.lpop(redis_title).decode('UTF-8') + api_key = redis_.lpop(redis_key_name_openaikey_list).decode('UTF-8') + # redis_title:{"id": id_, "title": title} + title_uuid_dict = json.loads(title_uuid_dict_str) + + title = title_uuid_dict["title"] + uuid_id = title_uuid_dict["id"] + + t = Thread(target=threading_mulu, args=(api_key, + title, + uuid_id, + )) + t.start() + else: + time.sleep(1) + continue + + +def threading_2(): + ''' + 顺序读取子任务 + :return: + ''' + while True: + if redis_.llen(redis_small_task) != 0 and redis_.llen(redis_key_name_openaikey_list) != 0: + # 执行小标题的任务 + api_key = redis_.lpop(redis_key_name_openaikey_list).decode('UTF-8') + small_title = redis_.lpop(redis_small_task).decode('UTF-8') + small_title = json.loads(small_title) + task_type = small_title["task_type"] + uuid = small_title["uuid"] + main_parameter = small_title["main_parameter"] + + # "task_type": "paper_content", + # "uuid": uuid, + # "main_parameter": [ + # "task_type": "paper_content", + # "task_type": "chat_abstract", + # "task_type": "kaitibaogao", + + if task_type == "kaitibaogao": + t = Thread(target=chat_kaitibaogao, args=(api_key, + uuid, + main_parameter, + task_type)) + t.start() + elif task_type == "chat_abstract": + t = Thread(target=chat_abstract_keyword, args=(api_key, + uuid, + main_parameter, + task_type)) + t.start() + elif task_type == "paper_content": + t = Thread(target=chat_content, args=(api_key, + uuid, + main_parameter, + task_type)) + t.start() + elif task_type == "thanks_task": + t = Thread(target=chat_thanks, args=(api_key, + uuid, + main_parameter, + task_type)) + t.start() + elif task_type == "references_task": + t = Thread(target=chat_references, args=(api_key, + uuid, + main_parameter, + task_type)) + t.start() + else: + time.sleep(1) + continue + + +def threading_3(): + while True: + res_end_list = [] + res_dict = redis_.hgetall(redis_res) + for key, values in res_dict.items(): + values_dict = json.loads(values) + # "num_small_task": len(small_task_list) - 1, + # "tasking_num": 0, + if int(values_dict["num_small_task"]) == int(values_dict["tasking_num"]): + res_end_list.append(key) + if res_end_list != []: + for key in res_end_list: + redis_.hdel(redis_res, key) + + res_str = res_dict[key].decode("utf-8") + json_str = json.dumps(res_str, indent=4, ensure_ascii=False) + + key = str(key, encoding="utf-8") + uuid_path = os.path.join(config.project_data_txt_path, key) + + os.makedirs(uuid_path) + + paper_content_path = os.path.join(uuid_path, "paper_content.json") + with open(paper_content_path, 'w') as json_file: + json_file.write(json_str) + + save_word_paper = os.path.join(uuid_path, "paper.docx") + save_word_paper_start = os.path.join(uuid_path, "paper_start.docx") + print("java_path", paper_content_path, ) + os.system( + "java -Dfile.encoding=UTF-8 -jar '/home/majiahui/ChatGPT_Sever/aiXieZuoPro.jar' '{}' '{}' '{}'".format( + paper_content_path, + save_word_paper, + save_word_paper_start)) + + url_path_paper = paper_download_url.format(key) + url_path_kaiti = paper_start_download_url.format(key) + return_text = str({"id": key, + "content_url_path": url_path_paper, + "content_report_url_path": url_path_kaiti, + "probabilities": None, + "status_code": 200}) + redis_.srem(redis_title_ing, key) + redis_.set(key, return_text, 28800) + + time.sleep(1) + + +# def main(title): +# # print(request.remote_addr) +# # title = request.json["title"] +# +# id_ = str(uuid.uuid1()) +# print(id_) +# redis_.rpush(redis_title, json.dumps({"id": id_, "title": title})) # 加入redis + + +@app.route("/chat", methods=["POST"]) +def chat(): + print(request.remote_addr) + title = request.json["title"] + id_ = str(uuid.uuid1()) + print(id_) + redis_.rpush(redis_title, json.dumps({"id": id_, "title": title},ensure_ascii=False)) # 加入redis + redis_.persist(redis_key_name_openaikey_list) + return_text = {"texts": {'id': id_, }, "probabilities": None, "status_code": 200} + print("ok") + redis_.sadd(redis_title_ing, id_) + + return jsonify(return_text) # 返回结果 + + +@app.route("/download", methods=['GET']) +def download_file(): + # 需要知道2个参数, 第1个参数是本地目录的path, 第2个参数是文件名(带扩展名) + # directory = os.path.join(project_data_txt_path, filename) # 假设在当前目录 + + # uuid_path, word_name = str(filename).split("/") + # word_path_root = os.path.join(project_data_txt_path, uuid_path) + # response = make_response(send_from_directory(word_path_root, word_name, as_attachment=True)) + # response.headers["Content-Disposition"] = "attachment; filename={}".format(filename.encode().decode('latin-1')) + filename_path = request.args.get('filename_path', '') + filename = filename_path.split("/")[1] + path_name = os.path.join(config.project_data_txt_path, filename_path) + with open(path_name, 'rb') as f: + stream = f.read() + response = Response(stream, content_type='application/octet-stream') + response.headers['Content-disposition'] = 'attachment; filename={}'.format(filename) + return response + + +@app.route("/search", methods=["POST"]) +def search(): + id_ = request.json['id'] # 获取用户query中的文本 例如"I love you" + result = redis_.get(id_) # 获取该query的模型结果 + + # if redis_.hexists(redis_key_name_openaikey_bad_dict, id_) == True: + # result_text = {'code': "204", 'text': "", 'probabilities': None} + if result is not None: + # redis_.delete(id_) + # result_dict = result.decode('UTF-8') + + result_dict = eval(result) + # return_text = {"id":query_id, "load_result_path": load_result_path, "probabilities": None, "status_code": 200} + query_id = result_dict["id"] + # "content_url_path": url_path_paper, + # "content_report_url_path": url_path_kaiti, + content_url_path = result_dict["content_url_path"] + content_report_url_path = result_dict["content_report_url_path"] + probabilities = result_dict["probabilities"] + result_text = {'code': 200, + 'content_url_path': content_url_path, + 'content_report_url_path': content_report_url_path, + 'probabilities': probabilities} + else: + querying_list = list(redis_.smembers(redis_title_ing)) + querying_set = set() + for i in querying_list: + querying_set.add(i.decode()) + + querying_bool = False + if id_ in querying_set: + querying_bool = True + + query_list_json = redis_.lrange(redis_title, 0, -1) + query_set_ids = set() + for i in query_list_json: + data_dict = json.loads(i) + query_id = data_dict['id'] + query_set_ids.add(query_id) + + query_bool = False + if id_ in query_set_ids: + query_bool = True + + if querying_bool == True and query_bool == True: + result_text = {'code': "201", 'text': "", 'probabilities': None} + elif querying_bool == True and query_bool == False: + result_text = {'code': "202", 'text': "", 'probabilities': None} + else: + result_text = {'code': "203", 'text': "", 'probabilities': None} + return jsonify(result_text) # 返回结果 + + +# threading_1 # 根据标题获取子任务，存入子任务序列 +# threading_2 # 根据子任务生成结果，存入结果序列 +# threading_3 # 根据存储的结果序列，看是否完成，如果完成输出json文件以及word +t = Thread(target=threading_1) +t.start() +t = Thread(target=threading_2) +t.start() +t = Thread(target=threading_3) +t.start() + +if __name__ == '__main__': + # main("大型商业建筑人员疏散设计研究") + app.run(host="0.0.0.0", port=config.flask_port, threaded=True, debug=False) \ No newline at end of file diff --git a/flask_serve_2.py b/flask_serve_2.py index addff58..399168a 100644 --- a/flask_serve_2.py +++ b/flask_serve_2.py @@ -14,9 +14,10 @@ import base64 import re import urllib.parse as pa import socket -from serve_config_1 import Config +from serve_config_2 import Config import requests + config = Config() app = Flask(__name__) @@ -27,14 +28,13 @@ s.connect(("8.8.8.8", 80)) localhostip = s.getsockname()[0] lock = threading.RLock() -pool = redis.ConnectionPool(host=config.reids_ip, port=config.reids_port, max_connections=50, db=config.reids_db, - password=config.reids_password) +pool = redis.ConnectionPool(host=config.reids_ip, port=config.reids_port, max_connections=50, db=config.reids_db, password=config.reids_password) redis_ = redis.Redis(connection_pool=pool, decode_responses=True) thanks = "致谢" references = "参考文献" -flask_serves_env = "http://{}:{}".format(localhostip, config.flask_port) +flask_serves_env = "http://{}:{}".format(localhostip,config.flask_port) paper_download_url = flask_serves_env + "/download?filename_path={}/paper.docx" paper_start_download_url = flask_serves_env + "/download?filename_path={}/paper_start.docx" @@ -76,14 +76,14 @@ def request_api_chatgpt(api_key, prompt): response = requests.post(url, headers=headers, data=json.dumps(data), - timeout=600) + timeout=240) return response +def chat_kaitibaogao(api_key, uuid, main_parameter,task_type): -def chat_kaitibaogao(api_key, uuid, main_parameter, task_type): try: - response = request_api_chatgpt(api_key, config.kaitibaogao_prompt.format(main_parameter[0])) + response =request_api_chatgpt(api_key, config.kaitibaogao_prompt.format(main_parameter[0])) res = response.json() kaitibaogao = res["choices"][0]["message"]["content"] redis_.rpush(redis_key_name_openaikey_list, api_key) @@ -94,13 +94,27 @@ def chat_kaitibaogao(api_key, uuid, main_parameter, task_type): 发送警报 """ kaitibaogao = "" - redis_.hset(redis_key_name_openaikey_bad_dict, uuid, str((api_key, task_type))) + + kaitibaogao_task = { + "task_type": task_type, + "uuid": uuid, + "main_parameter": main_parameter + } + time.sleep(3) + small_task = json.dumps(kaitibaogao_task, ensure_ascii=False) + redis_.lpush(redis_small_task, small_task) + + redis_.rpush(redis_key_name_openaikey_list, api_key) + redis_.persist(redis_key_name_openaikey_list) + + redis_.hset(redis_key_name_openaikey_bad_dict, uuid, str((api_key,task_type))) + return lock.acquire() res_dict_str = redis_.hget(redis_res, uuid) res_dict = json.loads(res_dict_str) res_dict["tasking_num"] += 1 - print("子任务进度".format(uuid), res_dict["tasking_num"]) + print("子任务进度".format(uuid),res_dict["tasking_num"]) res_dict["开题报告"] = kaitibaogao res_dict_str = json.dumps(res_dict, ensure_ascii=False) redis_.hset(redis_res, uuid, res_dict_str) @@ -108,10 +122,11 @@ def chat_kaitibaogao(api_key, uuid, main_parameter, task_type): def chat_abstract_keyword(api_key, uuid, main_parameter, task_type): + try: # 生成中文摘要 - response = request_api_chatgpt(api_key, config.chinese_abstract_prompt.format(main_parameter[0])) + response =request_api_chatgpt(api_key, config.chinese_abstract_prompt.format(main_parameter[0])) res = response.json() chinese_abstract = res["choices"][0]["message"]["content"] @@ -121,12 +136,14 @@ def chat_abstract_keyword(api_key, uuid, main_parameter, task_type): res = response.json() english_abstract = res["choices"][0]["message"]["content"] + # 生成中文关键字 response = request_api_chatgpt(api_key, config.chinese_keyword_prompt.format(chinese_abstract)) res = response.json() chinese_keyword = res["choices"][0]["message"]["content"] + # 生成英文关键字 response = request_api_chatgpt(api_key, config.english_keyword_prompt.format(chinese_keyword)) res = response.json() @@ -142,7 +159,21 @@ def chat_abstract_keyword(api_key, uuid, main_parameter, task_type): english_abstract = "" chinese_keyword = "" english_keyword = "" - redis_.hset(redis_key_name_openaikey_bad_dict, uuid, str((api_key, task_type))) + + chat_abstract_task = { + "task_type": task_type, + "uuid": uuid, + "main_parameter": main_parameter + } + time.sleep(3) + small_task = json.dumps(chat_abstract_task, ensure_ascii=False) + redis_.lpush(redis_small_task, small_task) + + redis_.rpush(redis_key_name_openaikey_list, api_key) + redis_.persist(redis_key_name_openaikey_list) + + redis_.hset(redis_key_name_openaikey_bad_dict, uuid, str((api_key,task_type))) + return paper_abstract_keyword = { "中文摘要": chinese_abstract, @@ -164,7 +195,7 @@ def chat_abstract_keyword(api_key, uuid, main_parameter, task_type): res_dict_str = redis_.hget(redis_res, uuid) res_dict = json.loads(res_dict_str) res_dict["tasking_num"] += 1 - print("子任务进度".format(uuid), res_dict["tasking_num"]) + print("子任务进度".format(uuid),res_dict["tasking_num"]) res_dict["中文摘要"] = paper_abstract_keyword["中文摘要"] res_dict["英文摘要"] = paper_abstract_keyword["英文摘要"] res_dict["中文关键词"] = paper_abstract_keyword["中文关键词"] @@ -190,6 +221,8 @@ def chat_content(api_key, uuid, main_parameter, task_type): if subtitle[:2] == "@@": res_content = subtitle[2:] + redis_.rpush(redis_key_name_openaikey_list, api_key) + redis_.persist(redis_key_name_openaikey_list) else: try: @@ -204,7 +237,20 @@ def chat_content(api_key, uuid, main_parameter, task_type): 发送警报 """ res_content = "" + paper_content = { + "task_type": "paper_content", + "uuid": uuid, + "main_parameter": main_parameter + } + time.sleep(3) + small_task = json.dumps(paper_content, ensure_ascii=False) + redis_.lpush(redis_small_task, small_task) + + redis_.rpush(redis_key_name_openaikey_list, api_key) + redis_.persist(redis_key_name_openaikey_list) + redis_.hset(redis_key_name_openaikey_bad_dict, uuid, str((api_key, task_type))) + return lock.acquire() res_dict_str = redis_.hget(redis_res, uuid) @@ -244,7 +290,20 @@ def chat_thanks(api_key, uuid, main_parameter, task_type): 发送警报 """ res_content = "" + thanks_task = { + "task_type": "thanks_task", + "uuid": uuid, + "main_parameter": main_parameter + } + time.sleep(3) + small_task = json.dumps(thanks_task, ensure_ascii=False) + redis_.lpush(redis_small_task, small_task) + + redis_.rpush(redis_key_name_openaikey_list, api_key) + redis_.persist(redis_key_name_openaikey_list) + redis_.hset(redis_key_name_openaikey_bad_dict, uuid, str((api_key, task_type))) + return lock.acquire() res_dict_str = redis_.hget(redis_res, uuid) @@ -284,7 +343,25 @@ def chat_references(api_key, uuid, main_parameter, task_type): 发送警报 """ res_content = "" + + references_task = { + "task_type": "references_task", + "uuid": uuid, + "main_parameter": [ + title, + mulu, + config.references_prompt + ] + } + time.sleep(3) + small_task = json.dumps(references_task, ensure_ascii=False) + redis_.lpush(redis_small_task, small_task) + + redis_.rpush(redis_key_name_openaikey_list, api_key) + redis_.persist(redis_key_name_openaikey_list) + redis_.hset(redis_key_name_openaikey_bad_dict, uuid, str((api_key, task_type))) + return # 加锁 读取resis并存储结果 lock.acquire() @@ -315,8 +392,15 @@ def threading_mulu(api_key, title, uuid): 发送警报 """ res_content = "" - redis_.hset(redis_key_name_openaikey_bad_dict, uuid, str(api_key, "mulu")) - mulu = "" + time.sleep(3) + redis_.lpush(redis_title, json.dumps({"id": uuid, "title": title}, ensure_ascii=False)) # 加入redis + redis_.persist(redis_key_name_openaikey_list) + + redis_.rpush(redis_key_name_openaikey_list, api_key) + redis_.persist(redis_key_name_openaikey_list) + + redis_.hset(redis_key_name_openaikey_bad_dict, uuid, str(api_key,"mulu")) + return try: mulu_list = str(mulu).split("\n") @@ -446,7 +530,7 @@ def threading_mulu(api_key, title, uuid): for small_task in small_task_list: small_task = json.dumps(small_task, ensure_ascii=False) redis_.rpush(redis_small_task, small_task) - redis_.persist(redis_key_name_openaikey_list) + res = { "uuid": uuid, @@ -586,11 +670,15 @@ def threading_3(): with open(paper_content_path, 'w') as json_file: json_file.write(json_str) - """ - 调用jar包 - 占位 + save_word_paper = os.path.join(uuid_path, "paper.docx") + save_word_paper_start = os.path.join(uuid_path, "paper_start.docx") + print("java_path", paper_content_path, ) + os.system( + "java -Dfile.encoding=UTF-8 -jar '/home/majiahui/ChatGPT_Sever/aiXieZuoPro.jar' '{}' '{}' '{}'".format( + paper_content_path, + save_word_paper, + save_word_paper_start)) - """ url_path_paper = paper_download_url.format(key) url_path_kaiti = paper_start_download_url.format(key) return_text = str({"id": key, @@ -619,7 +707,7 @@ def chat(): title = request.json["title"] id_ = str(uuid.uuid1()) print(id_) - redis_.rpush(redis_title, json.dumps({"id": id_, "title": title}, ensure_ascii=False)) # 加入redis + redis_.rpush(redis_title, json.dumps({"id": id_, "title": title},ensure_ascii=False)) # 加入redis redis_.persist(redis_key_name_openaikey_list) return_text = {"texts": {'id': id_, }, "probabilities": None, "status_code": 200} print("ok") @@ -651,25 +739,25 @@ def download_file(): def search(): id_ = request.json['id'] # 获取用户query中的文本 例如"I love you" result = redis_.get(id_) # 获取该query的模型结果 + + # if redis_.hexists(redis_key_name_openaikey_bad_dict, id_) == True: + # result_text = {'code': "204", 'text': "", 'probabilities': None} if result is not None: # redis_.delete(id_) # result_dict = result.decode('UTF-8') - if redis_.hexists(redis_key_name_openaikey_bad_dict, id_) == True: - result_text = {'code': "204", 'text': "", 'probabilities': None} - else: - result_dict = eval(result) - # return_text = {"id":query_id, "load_result_path": load_result_path, "probabilities": None, "status_code": 200} - query_id = result_dict["id"] - # "content_url_path": url_path_paper, - # "content_report_url_path": url_path_kaiti, - content_url_path = result_dict["content_url_path"] - content_report_url_path = result_dict["content_report_url_path"] - probabilities = result_dict["probabilities"] - result_text = {'code': 200, - 'content_url_path': content_url_path, - 'content_report_url_path': content_report_url_path, - 'probabilities': probabilities} + result_dict = eval(result) + # return_text = {"id":query_id, "load_result_path": load_result_path, "probabilities": None, "status_code": 200} + query_id = result_dict["id"] + # "content_url_path": url_path_paper, + # "content_report_url_path": url_path_kaiti, + content_url_path = result_dict["content_url_path"] + content_report_url_path = result_dict["content_report_url_path"] + probabilities = result_dict["probabilities"] + result_text = {'code': 200, + 'content_url_path': content_url_path, + 'content_report_url_path': content_report_url_path, + 'probabilities': probabilities} else: querying_list = list(redis_.smembers(redis_title_ing)) querying_set = set() @@ -703,12 +791,11 @@ def search(): # threading_1 # 根据标题获取子任务，存入子任务序列 # threading_2 # 根据子任务生成结果，存入结果序列 # threading_3 # 根据存储的结果序列，看是否完成，如果完成输出json文件以及word -t = Thread(target=threading_1) -t.start() + t = Thread(target=threading_2) t.start() if __name__ == '__main__': # main("大型商业建筑人员疏散设计研究") - app.run(host="0.0.0.0", port=14002, threaded=True, debug=False) \ No newline at end of file + app.run(host="0.0.0.0", port=config.flask_port, threaded=True, debug=False) \ No newline at end of file diff --git a/flask_serve_3.py b/flask_serve_3.py new file mode 100644 index 0000000..1be9342 --- /dev/null +++ b/flask_serve_3.py @@ -0,0 +1,724 @@ +from flask import Flask, jsonify, Response +from flask import request +import redis +import uuid +import json +import time +import threading +from threading import Thread +from flask import send_file, send_from_directory +import os +from flask import make_response +import openai +import base64 +import re +import urllib.parse as pa +import socket +from serve_config_1 import Config +import requests + + +config = Config() + +app = Flask(__name__) +app.config["JSON_AS_ASCII"] = False + +s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) +s.connect(("8.8.8.8", 80)) +localhostip = s.getsockname()[0] + +lock = threading.RLock() +pool = redis.ConnectionPool(host=config.reids_ip, port=config.reids_port, max_connections=50, db=config.reids_db, password=config.reids_password) +redis_ = redis.Redis(connection_pool=pool, decode_responses=True) + +thanks = "致谢" +references = "参考文献" + +flask_serves_env = "http://{}:{}".format(localhostip,config.flask_port) + +paper_download_url = flask_serves_env + "/download?filename_path={}/paper.docx" +paper_start_download_url = flask_serves_env + "/download?filename_path={}/paper_start.docx" + +redis_key_name_openaikey_bad_dict = "openaikey_bad_list_{}".format(str(localhostip)) + +redis_key_name_openaikey_list = "openaikey_list_{}".format(str(localhostip)) + +redis_title = "redis_title" + +redis_title_ing = "redis_title_ing" + +redis_small_task = "redis_small_task" + +redis_res = "redis_res" + +for i in config.openaikey_list: + redis_.rpush(redis_key_name_openaikey_list, i) + +redis_.hset(redis_key_name_openaikey_bad_dict, "1", "1") +redis_.persist(redis_key_name_openaikey_list) +redis_.persist(redis_key_name_openaikey_bad_dict) + + +def request_api_chatgpt(api_key, prompt): + OPENAI_API_KEY = api_key + url = "https://api.openai.com/v1/chat/completions" + headers = { + "Content-Type": "application/json", + "Authorization": f"Bearer {OPENAI_API_KEY}" + } + data = { + "model": "gpt-3.5-turbo", + "messages": [ + {"role": "user", "content": prompt}, + ], + "temperature": 0.5 + } + response = requests.post(url, + headers=headers, + data=json.dumps(data), + timeout=1200) + + return response + +def chat_kaitibaogao(api_key, uuid, main_parameter,task_type): + + try: + response =request_api_chatgpt(api_key, config.kaitibaogao_prompt.format(main_parameter[0])) + res = response.json() + kaitibaogao = res["choices"][0]["message"]["content"] + redis_.rpush(redis_key_name_openaikey_list, api_key) + redis_.persist(redis_key_name_openaikey_list) + + except: + """ + 发送警报 + """ + kaitibaogao = "" + redis_.hset(redis_key_name_openaikey_bad_dict, uuid, str((api_key,task_type))) + + + lock.acquire() + res_dict_str = redis_.hget(redis_res, uuid) + res_dict = json.loads(res_dict_str) + res_dict["tasking_num"] += 1 + print("子任务进度".format(uuid),res_dict["tasking_num"]) + res_dict["开题报告"] = kaitibaogao + res_dict_str = json.dumps(res_dict, ensure_ascii=False) + redis_.hset(redis_res, uuid, res_dict_str) + lock.release() + + +def chat_abstract_keyword(api_key, uuid, main_parameter, task_type): + + try: + # 生成中文摘要 + + response =request_api_chatgpt(api_key, config.chinese_abstract_prompt.format(main_parameter[0])) + res = response.json() + chinese_abstract = res["choices"][0]["message"]["content"] + + # 生成英文的摘要 + + response = request_api_chatgpt(api_key, config.english_abstract_prompt.format(chinese_abstract)) + res = response.json() + english_abstract = res["choices"][0]["message"]["content"] + + + # 生成中文关键字 + + response = request_api_chatgpt(api_key, config.chinese_keyword_prompt.format(chinese_abstract)) + res = response.json() + chinese_keyword = res["choices"][0]["message"]["content"] + + + # 生成英文关键字 + response = request_api_chatgpt(api_key, config.english_keyword_prompt.format(chinese_keyword)) + res = response.json() + english_keyword = res["choices"][0]["message"]["content"] + + redis_.rpush(redis_key_name_openaikey_list, api_key) + redis_.persist(redis_key_name_openaikey_list) + except: + """ + 发送警报 + """ + chinese_abstract = "" + english_abstract = "" + chinese_keyword = "" + english_keyword = "" + redis_.hset(redis_key_name_openaikey_bad_dict, uuid, str((api_key,task_type))) + + paper_abstract_keyword = { + "中文摘要": chinese_abstract, + "英文摘要": english_abstract, + "中文关键词": chinese_keyword, + "英文关键词": english_keyword + } + + # json_str = json.dumps(paper_abstract_keyword, indent=4, ensure_ascii=False) + # abstract_keyword_path = os.path.join(uuid_path, "abstract_keyword.json") + # with open(abstract_keyword_path, 'w') as json_file: + # json_file.write(json_str) + # + # lock.acquire() + # api_key_list.append(api_key) + # lock.release() + + lock.acquire() + res_dict_str = redis_.hget(redis_res, uuid) + res_dict = json.loads(res_dict_str) + res_dict["tasking_num"] += 1 + print("子任务进度".format(uuid),res_dict["tasking_num"]) + res_dict["中文摘要"] = paper_abstract_keyword["中文摘要"] + res_dict["英文摘要"] = paper_abstract_keyword["英文摘要"] + res_dict["中文关键词"] = paper_abstract_keyword["中文关键词"] + res_dict["英文关键词"] = paper_abstract_keyword["英文关键词"] + res_dict_str = json.dumps(res_dict, ensure_ascii=False) + redis_.hset(redis_res, uuid, res_dict_str) + lock.release() + + +def chat_content(api_key, uuid, main_parameter, task_type): + ''' + + :param api_key: + :param uuid: + :param main_parameter: + :return: + ''' + content_index = main_parameter[0] + title = main_parameter[1] + mulu = main_parameter[2] + subtitle = main_parameter[3] + prompt = main_parameter[4] + + if subtitle[:2] == "@@": + res_content = subtitle[2:] + redis_.rpush(redis_key_name_openaikey_list, api_key) + redis_.persist(redis_key_name_openaikey_list) + else: + try: + + response = request_api_chatgpt(api_key, prompt.format(title, mulu, subtitle)) + res = response.json() + res_content = res["choices"][0]["message"]["content"] + + redis_.rpush(redis_key_name_openaikey_list, api_key) + redis_.persist(redis_key_name_openaikey_list) + except: + """ + 发送警报 + """ + res_content = "" + redis_.hset(redis_key_name_openaikey_bad_dict, uuid, str((api_key,task_type))) + + lock.acquire() + res_dict_str = redis_.hget(redis_res, uuid) + res_dict = json.loads(res_dict_str) + res_dict["tasking_num"] += 1 + print("子任务进度".format(uuid), res_dict["tasking_num"]) + table_of_contents = res_dict["table_of_contents"] + table_of_contents[content_index] = res_content + res_dict["table_of_contents"] = table_of_contents + res_dict_str = json.dumps(res_dict, ensure_ascii=False) + redis_.hset(redis_res, uuid, res_dict_str) + lock.release() + + +def chat_thanks(api_key, uuid, main_parameter, task_type): + ''' + + :param api_key: + :param uuid: + :param main_parameter: + :return: + ''' + # title, + # thank_prompt + title = main_parameter[0] + prompt = main_parameter[1] + + try: + response = request_api_chatgpt(api_key, prompt.format(title)) + res = response.json() + res_content = res["choices"][0]["message"]["content"] + + redis_.rpush(redis_key_name_openaikey_list, api_key) + redis_.persist(redis_key_name_openaikey_list) + except: + """ + 发送警报 + """ + res_content = "" + redis_.hset(redis_key_name_openaikey_bad_dict, uuid, str((api_key,task_type))) + + lock.acquire() + res_dict_str = redis_.hget(redis_res, uuid) + res_dict = json.loads(res_dict_str) + res_dict["tasking_num"] += 1 + print("子任务进度".format(uuid), res_dict["tasking_num"]) + res_dict["致谢"] = res_content + res_dict_str = json.dumps(res_dict, ensure_ascii=False) + redis_.hset(redis_res, uuid, res_dict_str) + lock.release() + + +def chat_references(api_key, uuid, main_parameter, task_type): + ''' + + :param api_key: + :param uuid: + :param main_parameter: + :return: + ''' + # title, + # mulu, + # references_prompt + title = main_parameter[0] + mulu = main_parameter[1] + prompt = main_parameter[2] + try: + + response = request_api_chatgpt(api_key, prompt.format(title, mulu)) + res = response.json() + res_content = res["choices"][0]["message"]["content"] + + redis_.rpush(redis_key_name_openaikey_list, api_key) + redis_.persist(redis_key_name_openaikey_list) + except: + """ + 发送警报 + """ + res_content = "" + redis_.hset(redis_key_name_openaikey_bad_dict, uuid, str((api_key,task_type))) + + # 加锁 读取resis并存储结果 + lock.acquire() + res_dict_str = redis_.hget(redis_res, uuid) + res_dict = json.loads(res_dict_str) + res_dict["tasking_num"] += 1 + print("子任务进度".format(uuid), res_dict["tasking_num"]) + res_dict["参考文献"] = res_content + res_dict_str = json.dumps(res_dict, ensure_ascii=False) + redis_.hset(redis_res, uuid, res_dict_str) + lock.release() + + +def threading_mulu(api_key, title, uuid): + ''' + 生成目录并吧任务拆解进入子任务的redis_list中和储存结果的redis_list中 + :return: + ''' + try: + response = request_api_chatgpt(api_key, config.mulu_prompt.format(title)) + res = response.json() + mulu = res["choices"][0]["message"]["content"] + + redis_.rpush(redis_key_name_openaikey_list, api_key) + redis_.persist(redis_key_name_openaikey_list) + except: + """ + 发送警报 + """ + res_content = "" + redis_.hset(redis_key_name_openaikey_bad_dict, uuid, str(api_key,"mulu")) + mulu = "" + + try: + mulu_list = str(mulu).split("\n") + mulu_list = [i.strip() for i in mulu_list if i != ""] + + print(mulu_list) + + cun_bool = False + table_of_contents = [mulu_list[0]] + + for i in mulu_list[1:]: + result_second_biaoti_list = re.findall(config.pantten_second_biaoti, i) + result_other_biaoti_list = re.findall(config.pantten_other_biaoti, i) + if result_second_biaoti_list != []: + table_of_contents.append("@@" + i) + cun_bool = True + continue + if cun_bool == False: + continue + else: + if result_other_biaoti_list != []: + table_of_contents.append("@@" + i) + else: + table_of_contents.append(i) + + print(table_of_contents) + # table_of_contents = table_of_contents[:3] + table_of_contents[-1:] + # print(table_of_contents) + + thanks_references_bool_table = table_of_contents[-3:] + + # thanks = "致谢" + # references = "参考文献" + if references in thanks_references_bool_table: + table_of_contents.remove(references) + + if thanks in thanks_references_bool_table: + table_of_contents.remove(thanks) + + # table_of_contents.append(thanks) + # table_of_contents.append(references) + + # if thanks not in thanks_bool_table: + # table_of_contents.insert(-1, "致谢") + # + # if thanks not in thanks_bool_table: + # table_of_contents.insert(-1, "致谢") + + print(len(table_of_contents)) + + small_task_list = [] + # api_key, + # index, + # title, + # mulu, + # subtitle, + # prompt + kaitibaogao_task = { + "task_type": "kaitibaogao", + "uuid": uuid, + "main_parameter": [title] + } + + chat_abstract_task = { + "task_type": "chat_abstract", + "uuid": uuid, + "main_parameter": [title] + } + small_task_list.append(kaitibaogao_task) + small_task_list.append(chat_abstract_task) + content_index = 0 + while True: + if content_index == len(table_of_contents): + break + subtitle = table_of_contents[content_index] + if content_index == 0: + prompt = config.first_title_prompt + elif subtitle == "参考文献": + prompt = config.references_prompt + elif subtitle == "致谢": + prompt = config.thank_prompt + else: + prompt = config.small_title_prompt + print("请求的所有参数", + content_index, + title, + subtitle, + prompt) + + paper_content = { + "task_type": "paper_content", + "uuid": uuid, + "main_parameter": [ + content_index, + title, + mulu, + subtitle, + prompt + ] + } + + small_task_list.append(paper_content) + content_index += 1 + + thanks_task = { + "task_type": "thanks_task", + "uuid": uuid, + "main_parameter": [ + title, + config.thank_prompt + ] + } + + references_task = { + "task_type": "references_task", + "uuid": uuid, + "main_parameter": [ + title, + mulu, + config.references_prompt + ] + } + + small_task_list.append(thanks_task) + small_task_list.append(references_task) + + for small_task in small_task_list: + small_task = json.dumps(small_task, ensure_ascii=False) + redis_.rpush(redis_small_task, small_task) + + + res = { + "uuid": uuid, + "num_small_task": len(small_task_list), + "tasking_num": 0, + "标题": title, + "目录": mulu, + "开题报告": "", + "任务书": "", + "中文摘要": "", + "英文摘要": "", + "中文关键词": "", + "英文关键词": "", + "正文": "", + "致谢": "", + "参考文献": "", + "table_of_contents": [""] * len(table_of_contents) + } + + res = json.dumps(res, ensure_ascii=False) + redis_.hset(redis_res, uuid, res) + except: + print("目录程序错误") + + +def threading_1(): + # title, redis_key_name_openaikey_list + ''' + 生成目录 + :param title: + :param redis_key_name_openaikey_list: + :return: + ''' + while True: + if redis_.llen(redis_small_task) != 0: # 若队列中有元素就跳过 + time.sleep(1) + continue + elif redis_.llen(redis_title) != 0 and redis_.llen(redis_key_name_openaikey_list) != 0: + title_uuid_dict_str = redis_.lpop(redis_title).decode('UTF-8') + api_key = redis_.lpop(redis_key_name_openaikey_list).decode('UTF-8') + # redis_title:{"id": id_, "title": title} + title_uuid_dict = json.loads(title_uuid_dict_str) + + title = title_uuid_dict["title"] + uuid_id = title_uuid_dict["id"] + + t = Thread(target=threading_mulu, args=(api_key, + title, + uuid_id, + )) + t.start() + else: + time.sleep(1) + continue + + +def threading_2(): + ''' + 顺序读取子任务 + :return: + ''' + while True: + if redis_.llen(redis_small_task) != 0 and redis_.llen(redis_key_name_openaikey_list) != 0: + # 执行小标题的任务 + api_key = redis_.lpop(redis_key_name_openaikey_list).decode('UTF-8') + small_title = redis_.lpop(redis_small_task).decode('UTF-8') + small_title = json.loads(small_title) + task_type = small_title["task_type"] + uuid = small_title["uuid"] + main_parameter = small_title["main_parameter"] + + # "task_type": "paper_content", + # "uuid": uuid, + # "main_parameter": [ + # "task_type": "paper_content", + # "task_type": "chat_abstract", + # "task_type": "kaitibaogao", + + if task_type == "kaitibaogao": + t = Thread(target=chat_kaitibaogao, args=(api_key, + uuid, + main_parameter, + task_type)) + t.start() + elif task_type == "chat_abstract": + t = Thread(target=chat_abstract_keyword, args=(api_key, + uuid, + main_parameter, + task_type)) + t.start() + elif task_type == "paper_content": + t = Thread(target=chat_content, args=(api_key, + uuid, + main_parameter, + task_type)) + t.start() + elif task_type == "thanks_task": + t = Thread(target=chat_thanks, args=(api_key, + uuid, + main_parameter, + task_type)) + t.start() + elif task_type == "references_task": + t = Thread(target=chat_references, args=(api_key, + uuid, + main_parameter, + task_type)) + t.start() + else: + time.sleep(1) + continue + + +def threading_3(): + while True: + res_end_list = [] + res_dict = redis_.hgetall(redis_res) + for key, values in res_dict.items(): + values_dict = json.loads(values) + # "num_small_task": len(small_task_list) - 1, + # "tasking_num": 0, + if int(values_dict["num_small_task"]) == int(values_dict["tasking_num"]): + res_end_list.append(key) + if res_end_list != []: + for key in res_end_list: + redis_.hdel(redis_res, key) + + res_str = res_dict[key].decode("utf-8") + json_str = json.dumps(res_str, indent=4, ensure_ascii=False) + + key = str(key, encoding="utf-8") + uuid_path = os.path.join(config.project_data_txt_path, key) + + os.makedirs(uuid_path) + + paper_content_path = os.path.join(uuid_path, "paper_content.json") + with open(paper_content_path, 'w') as json_file: + json_file.write(json_str) + + save_word_paper = os.path.join(uuid_path, "paper.docx") + save_word_paper_start = os.path.join(uuid_path, "paper_start.docx") + print("java_path", paper_content_path, ) + os.system( + "java -Dfile.encoding=UTF-8 -jar '/home/majiahui/ChatGPT_Sever/aiXieZuoPro.jar' '{}' '{}' '{}'".format( + paper_content_path, + save_word_paper, + save_word_paper_start)) + + url_path_paper = paper_download_url.format(key) + url_path_kaiti = paper_start_download_url.format(key) + return_text = str({"id": key, + "content_url_path": url_path_paper, + "content_report_url_path": url_path_kaiti, + "probabilities": None, + "status_code": 200}) + redis_.srem(redis_title_ing, key) + redis_.set(key, return_text, 28800) + + time.sleep(1) + + +# def main(title): +# # print(request.remote_addr) +# # title = request.json["title"] +# +# id_ = str(uuid.uuid1()) +# print(id_) +# redis_.rpush(redis_title, json.dumps({"id": id_, "title": title})) # 加入redis + + +@app.route("/chat", methods=["POST"]) +def chat(): + print(request.remote_addr) + title = request.json["title"] + id_ = str(uuid.uuid1()) + print(id_) + redis_.rpush(redis_title, json.dumps({"id": id_, "title": title},ensure_ascii=False)) # 加入redis + redis_.persist(redis_key_name_openaikey_list) + return_text = {"texts": {'id': id_, }, "probabilities": None, "status_code": 200} + print("ok") + redis_.sadd(redis_title_ing, id_) + + return jsonify(return_text) # 返回结果 + + +@app.route("/download", methods=['GET']) +def download_file(): + # 需要知道2个参数, 第1个参数是本地目录的path, 第2个参数是文件名(带扩展名) + # directory = os.path.join(project_data_txt_path, filename) # 假设在当前目录 + + # uuid_path, word_name = str(filename).split("/") + # word_path_root = os.path.join(project_data_txt_path, uuid_path) + # response = make_response(send_from_directory(word_path_root, word_name, as_attachment=True)) + # response.headers["Content-Disposition"] = "attachment; filename={}".format(filename.encode().decode('latin-1')) + filename_path = request.args.get('filename_path', '') + filename = filename_path.split("/")[1] + path_name = os.path.join(config.project_data_txt_path, filename_path) + with open(path_name, 'rb') as f: + stream = f.read() + response = Response(stream, content_type='application/octet-stream') + response.headers['Content-disposition'] = 'attachment; filename={}'.format(filename) + return response + + +@app.route("/search", methods=["POST"]) +def search(): + id_ = request.json['id'] # 获取用户query中的文本 例如"I love you" + result = redis_.get(id_) # 获取该query的模型结果 + + if redis_.hexists(redis_key_name_openaikey_bad_dict, id_) == True: + result_text = {'code': "204", 'text': "", 'probabilities': None} + elif result is not None: + # redis_.delete(id_) + # result_dict = result.decode('UTF-8') + + result_dict = eval(result) + # return_text = {"id":query_id, "load_result_path": load_result_path, "probabilities": None, "status_code": 200} + query_id = result_dict["id"] + # "content_url_path": url_path_paper, + # "content_report_url_path": url_path_kaiti, + content_url_path = result_dict["content_url_path"] + content_report_url_path = result_dict["content_report_url_path"] + probabilities = result_dict["probabilities"] + result_text = {'code': 200, + 'content_url_path': content_url_path, + 'content_report_url_path': content_report_url_path, + 'probabilities': probabilities} + else: + querying_list = list(redis_.smembers(redis_title_ing)) + querying_set = set() + for i in querying_list: + querying_set.add(i.decode()) + + querying_bool = False + if id_ in querying_set: + querying_bool = True + + query_list_json = redis_.lrange(redis_title, 0, -1) + query_set_ids = set() + for i in query_list_json: + data_dict = json.loads(i) + query_id = data_dict['id'] + query_set_ids.add(query_id) + + query_bool = False + if id_ in query_set_ids: + query_bool = True + + if querying_bool == True and query_bool == True: + result_text = {'code': "201", 'text': "", 'probabilities': None} + elif querying_bool == True and query_bool == False: + result_text = {'code': "202", 'text': "", 'probabilities': None} + else: + result_text = {'code': "203", 'text': "", 'probabilities': None} + return jsonify(result_text) # 返回结果 + + +# threading_1 # 根据标题获取子任务，存入子任务序列 +# threading_2 # 根据子任务生成结果，存入结果序列 +# threading_3 # 根据存储的结果序列，看是否完成，如果完成输出json文件以及word +t = Thread(target=threading_1) +t.start() +t = Thread(target=threading_2) +t.start() + + +if __name__ == '__main__': + # main("大型商业建筑人员疏散设计研究") + app.run(host="0.0.0.0", port=config.flask_port, threaded=True, debug=False) \ No newline at end of file diff --git a/flask_sever_1.py b/flask_sever_1.py deleted file mode 100644 index 873a7ec..0000000 --- a/flask_sever_1.py +++ /dev/null @@ -1,719 +0,0 @@ -from flask import Flask, jsonify, Response -from flask import request -import redis -import uuid -import json -import time -import threading -from threading import Thread -from flask import send_file, send_from_directory -import os -from flask import make_response -import openai -import base64 -import re -import urllib.parse as pa -import socket -from serve_config_1 import Config -import requests - - -config = Config() - -app = Flask(__name__) -app.config["JSON_AS_ASCII"] = False - -s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) -s.connect(("8.8.8.8", 80)) -localhostip = s.getsockname()[0] - -lock = threading.RLock() -pool = redis.ConnectionPool(host=config.reids_ip, port=config.reids_port, max_connections=50, db=config.reids_db, password=config.reids_password) -redis_ = redis.Redis(connection_pool=pool, decode_responses=True) - -thanks = "致谢" -references = "参考文献" - -flask_serves_env = "http://{}:{}".format(localhostip,config.flask_port) - -paper_download_url = flask_serves_env + "/download?filename_path={}/paper.docx" -paper_start_download_url = flask_serves_env + "/download?filename_path={}/paper_start.docx" - -redis_key_name_openaikey_bad_dict = "openaikey_bad_list_{}".format(str(localhostip)) - -redis_key_name_openaikey_list = "openaikey_list_{}".format(str(localhostip)) - -redis_title = "redis_title" - -redis_title_ing = "redis_title_ing" - -redis_small_task = "redis_small_task" - -redis_res = "redis_res" - -for i in config.openaikey_list: - redis_.rpush(redis_key_name_openaikey_list, i) - -redis_.hset(redis_key_name_openaikey_bad_dict, "1", "1") -redis_.persist(redis_key_name_openaikey_list) -redis_.persist(redis_key_name_openaikey_bad_dict) - - -def request_api_chatgpt(api_key, prompt): - OPENAI_API_KEY = api_key - url = "https://api.openai.com/v1/chat/completions" - headers = { - "Content-Type": "application/json", - "Authorization": f"Bearer {OPENAI_API_KEY}" - } - data = { - "model": "gpt-3.5-turbo", - "messages": [ - {"role": "user", "content": prompt}, - ], - "temperature": 0.5 - } - response = requests.post(url, - headers=headers, - data=json.dumps(data), - timeout=600) - - return response - -def chat_kaitibaogao(api_key, uuid, main_parameter,task_type): - - try: - response =request_api_chatgpt(api_key, config.kaitibaogao_prompt.format(main_parameter[0])) - res = response.json() - kaitibaogao = res["choices"][0]["message"]["content"] - redis_.rpush(redis_key_name_openaikey_list, api_key) - redis_.persist(redis_key_name_openaikey_list) - - except: - """ - 发送警报 - """ - kaitibaogao = "" - redis_.hset(redis_key_name_openaikey_bad_dict, uuid, str((api_key,task_type))) - - - lock.acquire() - res_dict_str = redis_.hget(redis_res, uuid) - res_dict = json.loads(res_dict_str) - res_dict["tasking_num"] += 1 - print("子任务进度".format(uuid),res_dict["tasking_num"]) - res_dict["开题报告"] = kaitibaogao - res_dict_str = json.dumps(res_dict, ensure_ascii=False) - redis_.hset(redis_res, uuid, res_dict_str) - lock.release() - - -def chat_abstract_keyword(api_key, uuid, main_parameter, task_type): - - try: - # 生成中文摘要 - - response =request_api_chatgpt(api_key, config.chinese_abstract_prompt.format(main_parameter[0])) - res = response.json() - chinese_abstract = res["choices"][0]["message"]["content"] - - # 生成英文的摘要 - - response = request_api_chatgpt(api_key, config.english_abstract_prompt.format(chinese_abstract)) - res = response.json() - english_abstract = res["choices"][0]["message"]["content"] - - - # 生成中文关键字 - - response = request_api_chatgpt(api_key, config.chinese_keyword_prompt.format(chinese_abstract)) - res = response.json() - chinese_keyword = res["choices"][0]["message"]["content"] - - - # 生成英文关键字 - response = request_api_chatgpt(api_key, config.english_keyword_prompt.format(chinese_keyword)) - res = response.json() - english_keyword = res["choices"][0]["message"]["content"] - - redis_.rpush(redis_key_name_openaikey_list, api_key) - redis_.persist(redis_key_name_openaikey_list) - except: - """ - 发送警报 - """ - chinese_abstract = "" - english_abstract = "" - chinese_keyword = "" - english_keyword = "" - redis_.hset(redis_key_name_openaikey_bad_dict, uuid, str((api_key,task_type))) - - paper_abstract_keyword = { - "中文摘要": chinese_abstract, - "英文摘要": english_abstract, - "中文关键词": chinese_keyword, - "英文关键词": english_keyword - } - - # json_str = json.dumps(paper_abstract_keyword, indent=4, ensure_ascii=False) - # abstract_keyword_path = os.path.join(uuid_path, "abstract_keyword.json") - # with open(abstract_keyword_path, 'w') as json_file: - # json_file.write(json_str) - # - # lock.acquire() - # api_key_list.append(api_key) - # lock.release() - - lock.acquire() - res_dict_str = redis_.hget(redis_res, uuid) - res_dict = json.loads(res_dict_str) - res_dict["tasking_num"] += 1 - print("子任务进度".format(uuid),res_dict["tasking_num"]) - res_dict["中文摘要"] = paper_abstract_keyword["中文摘要"] - res_dict["英文摘要"] = paper_abstract_keyword["英文摘要"] - res_dict["中文关键词"] = paper_abstract_keyword["中文关键词"] - res_dict["英文关键词"] = paper_abstract_keyword["英文关键词"] - res_dict_str = json.dumps(res_dict, ensure_ascii=False) - redis_.hset(redis_res, uuid, res_dict_str) - lock.release() - - -def chat_content(api_key, uuid, main_parameter, task_type): - ''' - - :param api_key: - :param uuid: - :param main_parameter: - :return: - ''' - content_index = main_parameter[0] - title = main_parameter[1] - mulu = main_parameter[2] - subtitle = main_parameter[3] - prompt = main_parameter[4] - - if subtitle[:2] == "@@": - res_content = subtitle[2:] - else: - try: - - response = request_api_chatgpt(api_key, prompt.format(title, mulu, subtitle)) - res = response.json() - res_content = res["choices"][0]["message"]["content"] - - redis_.rpush(redis_key_name_openaikey_list, api_key) - redis_.persist(redis_key_name_openaikey_list) - except: - """ - 发送警报 - """ - res_content = "" - redis_.hset(redis_key_name_openaikey_bad_dict, uuid, str((api_key,task_type))) - - lock.acquire() - res_dict_str = redis_.hget(redis_res, uuid) - res_dict = json.loads(res_dict_str) - res_dict["tasking_num"] += 1 - print("子任务进度".format(uuid), res_dict["tasking_num"]) - table_of_contents = res_dict["table_of_contents"] - table_of_contents[content_index] = res_content - res_dict["table_of_contents"] = table_of_contents - res_dict_str = json.dumps(res_dict, ensure_ascii=False) - redis_.hset(redis_res, uuid, res_dict_str) - lock.release() - - -def chat_thanks(api_key, uuid, main_parameter, task_type): - ''' - - :param api_key: - :param uuid: - :param main_parameter: - :return: - ''' - # title, - # thank_prompt - title = main_parameter[0] - prompt = main_parameter[1] - - try: - response = request_api_chatgpt(api_key, prompt.format(title)) - res = response.json() - res_content = res["choices"][0]["message"]["content"] - - redis_.rpush(redis_key_name_openaikey_list, api_key) - redis_.persist(redis_key_name_openaikey_list) - except: - """ - 发送警报 - """ - res_content = "" - redis_.hset(redis_key_name_openaikey_bad_dict, uuid, str((api_key,task_type))) - - lock.acquire() - res_dict_str = redis_.hget(redis_res, uuid) - res_dict = json.loads(res_dict_str) - res_dict["tasking_num"] += 1 - print("子任务进度".format(uuid), res_dict["tasking_num"]) - res_dict["致谢"] = res_content - res_dict_str = json.dumps(res_dict, ensure_ascii=False) - redis_.hset(redis_res, uuid, res_dict_str) - lock.release() - - -def chat_references(api_key, uuid, main_parameter, task_type): - ''' - - :param api_key: - :param uuid: - :param main_parameter: - :return: - ''' - # title, - # mulu, - # references_prompt - title = main_parameter[0] - mulu = main_parameter[1] - prompt = main_parameter[2] - try: - - response = request_api_chatgpt(api_key, prompt.format(title, mulu)) - res = response.json() - res_content = res["choices"][0]["message"]["content"] - - redis_.rpush(redis_key_name_openaikey_list, api_key) - redis_.persist(redis_key_name_openaikey_list) - except: - """ - 发送警报 - """ - res_content = "" - redis_.hset(redis_key_name_openaikey_bad_dict, uuid, str((api_key,task_type))) - - # 加锁 读取resis并存储结果 - lock.acquire() - res_dict_str = redis_.hget(redis_res, uuid) - res_dict = json.loads(res_dict_str) - res_dict["tasking_num"] += 1 - print("子任务进度".format(uuid), res_dict["tasking_num"]) - res_dict["参考文献"] = res_content - res_dict_str = json.dumps(res_dict, ensure_ascii=False) - redis_.hset(redis_res, uuid, res_dict_str) - lock.release() - - -def threading_mulu(api_key, title, uuid): - ''' - 生成目录并吧任务拆解进入子任务的redis_list中和储存结果的redis_list中 - :return: - ''' - try: - response = request_api_chatgpt(api_key, config.mulu_prompt.format(title)) - res = response.json() - mulu = res["choices"][0]["message"]["content"] - - redis_.rpush(redis_key_name_openaikey_list, api_key) - redis_.persist(redis_key_name_openaikey_list) - except: - """ - 发送警报 - """ - res_content = "" - redis_.hset(redis_key_name_openaikey_bad_dict, uuid, str(api_key,"mulu")) - mulu = "" - - try: - mulu_list = str(mulu).split("\n") - mulu_list = [i.strip() for i in mulu_list if i != ""] - - print(mulu_list) - - cun_bool = False - table_of_contents = [mulu_list[0]] - - for i in mulu_list[1:]: - result_second_biaoti_list = re.findall(config.pantten_second_biaoti, i) - result_other_biaoti_list = re.findall(config.pantten_other_biaoti, i) - if result_second_biaoti_list != []: - table_of_contents.append("@@" + i) - cun_bool = True - continue - if cun_bool == False: - continue - else: - if result_other_biaoti_list != []: - table_of_contents.append("@@" + i) - else: - table_of_contents.append(i) - - print(table_of_contents) - # table_of_contents = table_of_contents[:3] + table_of_contents[-1:] - # print(table_of_contents) - - thanks_references_bool_table = table_of_contents[-3:] - - # thanks = "致谢" - # references = "参考文献" - if references in thanks_references_bool_table: - table_of_contents.remove(references) - - if thanks in thanks_references_bool_table: - table_of_contents.remove(thanks) - - # table_of_contents.append(thanks) - # table_of_contents.append(references) - - # if thanks not in thanks_bool_table: - # table_of_contents.insert(-1, "致谢") - # - # if thanks not in thanks_bool_table: - # table_of_contents.insert(-1, "致谢") - - print(len(table_of_contents)) - - small_task_list = [] - # api_key, - # index, - # title, - # mulu, - # subtitle, - # prompt - kaitibaogao_task = { - "task_type": "kaitibaogao", - "uuid": uuid, - "main_parameter": [title] - } - - chat_abstract_task = { - "task_type": "chat_abstract", - "uuid": uuid, - "main_parameter": [title] - } - small_task_list.append(kaitibaogao_task) - small_task_list.append(chat_abstract_task) - content_index = 0 - while True: - if content_index == len(table_of_contents): - break - subtitle = table_of_contents[content_index] - if content_index == 0: - prompt = config.first_title_prompt - elif subtitle == "参考文献": - prompt = config.references_prompt - elif subtitle == "致谢": - prompt = config.thank_prompt - else: - prompt = config.small_title_prompt - print("请求的所有参数", - content_index, - title, - subtitle, - prompt) - - paper_content = { - "task_type": "paper_content", - "uuid": uuid, - "main_parameter": [ - content_index, - title, - mulu, - subtitle, - prompt - ] - } - - small_task_list.append(paper_content) - content_index += 1 - - thanks_task = { - "task_type": "thanks_task", - "uuid": uuid, - "main_parameter": [ - title, - config.thank_prompt - ] - } - - references_task = { - "task_type": "references_task", - "uuid": uuid, - "main_parameter": [ - title, - mulu, - config.references_prompt - ] - } - - small_task_list.append(thanks_task) - small_task_list.append(references_task) - - for small_task in small_task_list: - small_task = json.dumps(small_task, ensure_ascii=False) - redis_.rpush(redis_small_task, small_task) - redis_.persist(redis_key_name_openaikey_list) - - res = { - "uuid": uuid, - "num_small_task": len(small_task_list), - "tasking_num": 0, - "标题": title, - "目录": mulu, - "开题报告": "", - "任务书": "", - "中文摘要": "", - "英文摘要": "", - "中文关键词": "", - "英文关键词": "", - "正文": "", - "致谢": "", - "参考文献": "", - "table_of_contents": [""] * len(table_of_contents) - } - - res = json.dumps(res, ensure_ascii=False) - redis_.hset(redis_res, uuid, res) - except: - print("目录程序错误") - - -def threading_1(): - # title, redis_key_name_openaikey_list - ''' - 生成目录 - :param title: - :param redis_key_name_openaikey_list: - :return: - ''' - while True: - if redis_.llen(redis_small_task) != 0: # 若队列中有元素就跳过 - time.sleep(1) - continue - elif redis_.llen(redis_title) != 0 and redis_.llen(redis_key_name_openaikey_list) != 0: - title_uuid_dict_str = redis_.lpop(redis_title).decode('UTF-8') - api_key = redis_.lpop(redis_key_name_openaikey_list).decode('UTF-8') - # redis_title:{"id": id_, "title": title} - title_uuid_dict = json.loads(title_uuid_dict_str) - - title = title_uuid_dict["title"] - uuid_id = title_uuid_dict["id"] - - t = Thread(target=threading_mulu, args=(api_key, - title, - uuid_id, - )) - t.start() - else: - time.sleep(1) - continue - - -def threading_2(): - ''' - 顺序读取子任务 - :return: - ''' - while True: - if redis_.llen(redis_small_task) != 0 and redis_.llen(redis_key_name_openaikey_list) != 0: - # 执行小标题的任务 - api_key = redis_.lpop(redis_key_name_openaikey_list).decode('UTF-8') - small_title = redis_.lpop(redis_small_task).decode('UTF-8') - small_title = json.loads(small_title) - task_type = small_title["task_type"] - uuid = small_title["uuid"] - main_parameter = small_title["main_parameter"] - - # "task_type": "paper_content", - # "uuid": uuid, - # "main_parameter": [ - # "task_type": "paper_content", - # "task_type": "chat_abstract", - # "task_type": "kaitibaogao", - - if task_type == "kaitibaogao": - t = Thread(target=chat_kaitibaogao, args=(api_key, - uuid, - main_parameter, - task_type)) - t.start() - elif task_type == "chat_abstract": - t = Thread(target=chat_abstract_keyword, args=(api_key, - uuid, - main_parameter, - task_type)) - t.start() - elif task_type == "paper_content": - t = Thread(target=chat_content, args=(api_key, - uuid, - main_parameter, - task_type)) - t.start() - elif task_type == "thanks_task": - t = Thread(target=chat_thanks, args=(api_key, - uuid, - main_parameter, - task_type)) - t.start() - elif task_type == "references_task": - t = Thread(target=chat_references, args=(api_key, - uuid, - main_parameter, - task_type)) - t.start() - else: - time.sleep(1) - continue - - -def threading_3(): - while True: - res_end_list = [] - res_dict = redis_.hgetall(redis_res) - for key, values in res_dict.items(): - values_dict = json.loads(values) - # "num_small_task": len(small_task_list) - 1, - # "tasking_num": 0, - if int(values_dict["num_small_task"]) == int(values_dict["tasking_num"]): - res_end_list.append(key) - if res_end_list != []: - for key in res_end_list: - redis_.hdel(redis_res, key) - - res_str = res_dict[key].decode("utf-8") - json_str = json.dumps(res_str, indent=4, ensure_ascii=False) - - key = str(key, encoding="utf-8") - uuid_path = os.path.join(config.project_data_txt_path, key) - - os.makedirs(uuid_path) - - paper_content_path = os.path.join(uuid_path, "paper_content.json") - with open(paper_content_path, 'w') as json_file: - json_file.write(json_str) - - """ - 调用jar包 - 占位 - - """ - url_path_paper = paper_download_url.format(key) - url_path_kaiti = paper_start_download_url.format(key) - return_text = str({"id": key, - "content_url_path": url_path_paper, - "content_report_url_path": url_path_kaiti, - "probabilities": None, - "status_code": 200}) - redis_.srem(redis_title_ing, key) - redis_.set(key, return_text, 28800) - - time.sleep(1) - - -# def main(title): -# # print(request.remote_addr) -# # title = request.json["title"] -# -# id_ = str(uuid.uuid1()) -# print(id_) -# redis_.rpush(redis_title, json.dumps({"id": id_, "title": title})) # 加入redis - - -@app.route("/chat", methods=["POST"]) -def chat(): - print(request.remote_addr) - title = request.json["title"] - id_ = str(uuid.uuid1()) - print(id_) - redis_.rpush(redis_title, json.dumps({"id": id_, "title": title},ensure_ascii=False)) # 加入redis - redis_.persist(redis_key_name_openaikey_list) - return_text = {"texts": {'id': id_, }, "probabilities": None, "status_code": 200} - print("ok") - redis_.sadd(redis_title_ing, id_) - - return jsonify(return_text) # 返回结果 - - -@app.route("/download", methods=['GET']) -def download_file(): - # 需要知道2个参数, 第1个参数是本地目录的path, 第2个参数是文件名(带扩展名) - # directory = os.path.join(project_data_txt_path, filename) # 假设在当前目录 - - # uuid_path, word_name = str(filename).split("/") - # word_path_root = os.path.join(project_data_txt_path, uuid_path) - # response = make_response(send_from_directory(word_path_root, word_name, as_attachment=True)) - # response.headers["Content-Disposition"] = "attachment; filename={}".format(filename.encode().decode('latin-1')) - filename_path = request.args.get('filename_path', '') - filename = filename_path.split("/")[1] - path_name = os.path.join(config.project_data_txt_path, filename_path) - with open(path_name, 'rb') as f: - stream = f.read() - response = Response(stream, content_type='application/octet-stream') - response.headers['Content-disposition'] = 'attachment; filename={}'.format(filename) - return response - - -@app.route("/search", methods=["POST"]) -def search(): - id_ = request.json['id'] # 获取用户query中的文本 例如"I love you" - result = redis_.get(id_) # 获取该query的模型结果 - if result is not None: - # redis_.delete(id_) - # result_dict = result.decode('UTF-8') - if redis_.hexists(redis_key_name_openaikey_bad_dict, id_) == True: - result_text = {'code': "204", 'text': "", 'probabilities': None} - - else: - result_dict = eval(result) - # return_text = {"id":query_id, "load_result_path": load_result_path, "probabilities": None, "status_code": 200} - query_id = result_dict["id"] - # "content_url_path": url_path_paper, - # "content_report_url_path": url_path_kaiti, - content_url_path = result_dict["content_url_path"] - content_report_url_path = result_dict["content_report_url_path"] - probabilities = result_dict["probabilities"] - result_text = {'code': 200, - 'content_url_path': content_url_path, - 'content_report_url_path': content_report_url_path, - 'probabilities': probabilities} - else: - querying_list = list(redis_.smembers(redis_title_ing)) - querying_set = set() - for i in querying_list: - querying_set.add(i.decode()) - - querying_bool = False - if id_ in querying_set: - querying_bool = True - - query_list_json = redis_.lrange(redis_title, 0, -1) - query_set_ids = set() - for i in query_list_json: - data_dict = json.loads(i) - query_id = data_dict['id'] - query_set_ids.add(query_id) - - query_bool = False - if id_ in query_set_ids: - query_bool = True - - if querying_bool == True and query_bool == True: - result_text = {'code': "201", 'text': "", 'probabilities': None} - elif querying_bool == True and query_bool == False: - result_text = {'code': "202", 'text': "", 'probabilities': None} - else: - result_text = {'code': "203", 'text': "", 'probabilities': None} - return jsonify(result_text) # 返回结果 - - -# threading_1 # 根据标题获取子任务，存入子任务序列 -# threading_2 # 根据子任务生成结果，存入结果序列 -# threading_3 # 根据存储的结果序列，看是否完成，如果完成输出json文件以及word -t = Thread(target=threading_1) -t.start() -t = Thread(target=threading_2) -t.start() -t = Thread(target=threading_3) -t.start() - -if __name__ == '__main__': - # main("大型商业建筑人员疏散设计研究") - app.run(host="0.0.0.0", port=14002, threaded=True, debug=False) \ No newline at end of file diff --git a/ip对应关系 b/ip对应关系 new file mode 100644 index 0000000..7903c1a --- /dev/null +++ b/ip对应关系 @@ -0,0 +1,3 @@ +国外服务器1 104.244.89.190 +国外服务器2 107.182.189.58 +国外服务器3 104.244.90.248 \ No newline at end of file diff --git a/jar_bagk.py b/jar_bagk.py index 2b9bc51..88f2ebd 100644 --- a/jar_bagk.py +++ b/jar_bagk.py @@ -26,3 +26,8 @@ os.system("java -Dfile.encoding=UTF-8 -jar '/home/majiahui/ChatGPT_Sever/createA os.system("java -Dfile.encoding=UTF-8 -jar '/home/majiahui/ChatGPT_Sever/createAiXieZuoKaitiWord.jar' '{}' '{}'".format(kaitibaogao, save_word_kaiti)) + + +os.system("java -Dfile.encoding=UTF-8 -jar '/home/majiahui/ChatGPT_Sever/aiXieZuoPro.jar' '${data.json}' '${paperSavePath}' '${kaitiSavePath}'".format(content, + save_word, + save_word_kaiti)) diff --git a/run_serves.sh b/run_serves.sh new file mode 100644 index 0000000..19646ac --- /dev/null +++ b/run_serves.sh @@ -0,0 +1 @@ +nohup python flask_serve_1.py > myout.file_1 2>&1 & \ No newline at end of file diff --git a/serve_config_1.py b/serve_config_1.py index 98ed15f..5363122 100644 --- a/serve_config_1.py +++ b/serve_config_1.py @@ -1,45 +1,40 @@ - - class Config: def __init__(self): - # 目录提取拼接相关参数 self.pantten_second_biaoti = '[2二ⅡⅠ][、.]\s{0,}?[\u4e00-\u9fa5]+' self.pantten_other_biaoti = '[2-9二三四五六七八九ⅡⅢⅣⅤⅥⅦⅧⅨ][、.]\s{0,}?[\u4e00-\u9fa5]+' # chatgpt 接口相关参数 self.mulu_prompt = "请帮我根据题目为“{}”生成一个论文目录" - self.first_title_prompt = "论文题目是“{}”，目录是“{}”，请把其中的大标题“{}”的内容补充完整，补充内容字数在100字左右" - self.small_title_prompt = "论文题目是“{}”，目录是“{}”，请把其中的小标题“{}”的内容补充完整，补充内容字数在100字左右" + self.first_title_prompt = "论文题目是“{}”，目录是“{}”，请把其中的大标题“{}”的内容补充完整，补充内容字数在800字左右" + self.small_title_prompt = "论文题目是“{}”，目录是“{}”，请把其中的小标题“{}”的内容补充完整，补充内容字数在800字左右" self.references_prompt = "论文题目是“{}”，目录是“{}”，请为这篇论文生成15篇左右的参考文献，要求其中有有中文参考文献不低于12篇，英文参考文献不低于2篇" self.thank_prompt = "请以“{}”为题写一篇论文的致谢" - self.kaitibaogao_prompt = "请以《{}》为题目生成研究的主要的内容、背景、目的、意义，要求不少于100字" - self.chinese_abstract_prompt = "请以《{}》为题目生成论文摘要，要求生成的字数在100字左右" + self.kaitibaogao_prompt = "请以《{}》为题目生成研究的主要的内容、背景、目的、意义，要求不少于1500字" + self.chinese_abstract_prompt = "请以《{}》为题目生成论文摘要，要求生成的字数在800字左右" self.english_abstract_prompt = "请把“{}”这段文字翻译成英文" self.chinese_keyword_prompt = "请为“{}”这段论文摘要生成3-5个关键字" self.english_keyword_prompt = "请把“{}”这几个关键字翻译成英文" self.dabiaoti = ["二", "三", "四", "五", "六", "七", "八", "九"] self.project_data_txt_path = "/home/majiahui/ChatGPT_Sever/new_data_txt" - self.openaikey_list = ["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", - "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", - "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" + self.openaikey_list = ["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", + "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", + "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", + "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", + "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", + "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", + "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", ] - # "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", - # "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" - # "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", - # "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", - # "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", - # "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" # 流程相关参数 self.thanks = "致谢" self.references = "参考文献" # flask port - self.flask_port = "14000" + self.flask_port = "14002" # redis config self.reids_ip = 'localhost' self.reids_port = 63179 self.reids_db = 2 - self.reids_password='Zhicheng123*' \ No newline at end of file + self.reids_password = 'Zhicheng123*' diff --git a/serve_config_2.py b/serve_config_2.py index d7aa329..0938558 100644 --- a/serve_config_2.py +++ b/serve_config_2.py @@ -21,13 +21,15 @@ class Config: self.dabiaoti = ["二", "三", "四", "五", "六", "七", "八", "九"] self.project_data_txt_path = "/home/majiahui/ChatGPT_Sever/new_data_txt" self.openaikey_list = [ - "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", - "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", - "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" + "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", + "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", + "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", + "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", + "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", + "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", + "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", ] - # "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", - # "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", - # "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" + # 流程相关参数 self.thanks = "致谢" diff --git a/serve_config_3.py b/serve_config_3.py new file mode 100644 index 0000000..1b3ff70 --- /dev/null +++ b/serve_config_3.py @@ -0,0 +1,39 @@ + + +class Config: + def __init__(self): + + # 目录提取拼接相关参数 + self.pantten_second_biaoti = '[2二ⅡⅠ][、.]\s{0,}?[\u4e00-\u9fa5]+' + self.pantten_other_biaoti = '[2-9二三四五六七八九ⅡⅢⅣⅤⅥⅦⅧⅨ][、.]\s{0,}?[\u4e00-\u9fa5]+' + + # chatgpt 接口相关参数 + self.mulu_prompt = "请帮我根据题目为“{}”生成一个论文目录" + self.first_title_prompt = "论文题目是“{}”，目录是“{}”，请把其中的大标题“{}”的内容补充完整，补充内容字数在100字左右" + self.small_title_prompt = "论文题目是“{}”，目录是“{}”，请把其中的小标题“{}”的内容补充完整，补充内容字数在100字左右" + self.references_prompt = "论文题目是“{}”，目录是“{}”，请为这篇论文生成15篇左右的参考文献，要求其中有有中文参考文献不低于12篇，英文参考文献不低于2篇" + self.thank_prompt = "请以“{}”为题写一篇论文的致谢" + self.kaitibaogao_prompt = "请以《{}》为题目生成研究的主要的内容、背景、目的、意义，要求不少于100字" + self.chinese_abstract_prompt = "请以《{}》为题目生成论文摘要，要求生成的字数在100字左右" + self.english_abstract_prompt = "请把“{}”这段文字翻译成英文" + self.chinese_keyword_prompt = "请为“{}”这段论文摘要生成3-5个关键字" + self.english_keyword_prompt = "请把“{}”这几个关键字翻译成英文" + self.dabiaoti = ["二", "三", "四", "五", "六", "七", "八", "九"] + self.project_data_txt_path = "/home/majiahui/ChatGPT_Sever/new_data_txt" + self.openaikey_list = [ + + ] + + + # 流程相关参数 + self.thanks = "致谢" + self.references = "参考文献" + + # flask port + self.flask_port = "14000" + + # redis config + self.reids_ip = '104.244.89.190' + self.reids_port = 63179 + self.reids_db = 2 + self.reids_password='Zhicheng123*' \ No newline at end of file diff --git a/查询uuid.py b/查询uuid.py index 518b062..5bba47e 100644 --- a/查询uuid.py +++ b/查询uuid.py @@ -1,7 +1,7 @@ import requests import time -data = {"id": "a259c76a-d521-11ed-b23c-aaaa001b4bbf"} +data = {"id": "d5b4c5d2-d77a-11ed-981d-aaaa001b4bbf"} start = time.time() @@ -25,3 +25,25 @@ b689132c-d516-11ed-b4e7-aaaa001b4bbf c3f4bea8-d516-11ed-beda-aaaa001b4bbf cd948cb8-d516-11ed-9847-aaaa001b4bbf f2c5920c-d516-11ed-96b1-aaaa001b4bbf''' + + +''' +69b518cc-d677-11ed-9015-aaaa001b4bbf +''' + + +''' +1b07d42e-d74b-11ed-be5d-aaaa001b4bbf + +''' + + + +''' +ab184bce-d774-11ed-9b53-aaaa001b4bbf +ab18fae2-d774-11ed-85b9-aaaa001b4bbf +ab19a88e-d774-11ed-a75c-aaaa001b4bbf +ab1a5838-d774-11ed-bdd5-aaaa001b4bbf +ab1afd4c-d774-11ed-a3ef-aaaa001b4bbf + +''' \ No newline at end of file diff --git a/测试生成uuid.py b/测试生成uuid.py index f7e7d79..298baca 100644 --- a/测试生成uuid.py +++ b/测试生成uuid.py @@ -3,11 +3,12 @@ import time data = {"title": "大型商业建筑人员疏散设计研究"} -start = time.time() +for i in range(1): + start = time.time() -res = requests.post('http://104.244.89.190:14002/chat', json=data) -end = time.time() -print(end - start) + res = requests.post('http://104.244.89.190:14002/chat', json=data) + end = time.time() + print(end - start) -print(res.text) \ No newline at end of file + print(res.text) \ No newline at end of file