diff --git a/flask_serce_dev_1.py b/flask_serce_dev_1.py new file mode 100644 index 0000000..13e2549 --- /dev/null +++ b/flask_serce_dev_1.py @@ -0,0 +1,531 @@ +from flask import Flask, jsonify, Response +from flask import request +import redis +import uuid +import json +import time +import threading +from threading import Thread +from flask import send_file, send_from_directory +import os +from flask import make_response +import openai +import base64 +import re +import urllib.parse as pa +import socket + +s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) +s.connect(("8.8.8.8", 80)) +localhostip = s.getsockname()[0] + +lock = threading.RLock() +pool = redis.ConnectionPool(host='localhost', port=6379, max_connections=50, db=2) +redis_ = redis.Redis(connection_pool=pool, decode_responses=True) + +pantten_second_biaoti = '[2二ⅡⅠ][、.]\s{0,}?[\u4e00-\u9fa5]+' +pantten_other_biaoti = '[2-9二三四五六七八九ⅡⅢⅣⅤⅥⅦⅧⅨ][、.]\s{0,}?[\u4e00-\u9fa5]+' + +mulu_prompt = "请帮我根据题目为“{}”生成一个论文目录" +first_title_prompt = "论文题目是“{}”,目录是“{}”,请把其中的大标题“{}”的内容续写完整,保证续写内容不少于800字" +small_title_prompt = "论文题目是“{}”,目录是“{}”,请把其中的小标题“{}”的内容续写完整,保证续写内容不少于800字" +references_prompt = "论文题目是“{}”,目录是“{}”,请为这篇论文生成15篇中文的{},要求其中有有中文参考文献不低于12篇,英文参考文献不低于2篇" +thank_prompt = "论文题目是“{}”,目录是“{}”,请把其中的{}部分续写完整" +kaitibaogao_prompt = "请以《{}》为题目生成研究的主要的内容、背景、目的、意义,要求不少于100字" +chinese_abstract_prompt = "请以《{}》为题目生成论文摘要,要求不少于1500字" +english_abstract_prompt = "请把“{}”这段文字翻译成英文" +chinese_keyword_prompt = "请为“{}”这段论文摘要生成3-5个关键字" +english_keyword_prompt = "请把“{}”这几个关键字翻译成英文" +thanks = "致谢" +references = "参考文献" +dabiaoti = ["二", "三", "四", "五", "六", "七", "八", "九"] +project_data_txt_path = "/home/majiahui/ChatGPT_Sever/new_data_txt" + +""" + +key_list = [ + {"ip": key-api}, + {"ip": key-api}, + {"ip": key-api}, + ] + +redis_title = [] + +redis_title_ing = [] + +redis_small_task = [ + { + uuid, + api_key, + mulu_title_id, + title, + mulu, + subtitle, + prompt + } + ] + +redis_res = [ + { + "uuid": + "完成进度": + "标题": + "中文摘要":"", + "英文摘要" + "中文关键字" + "英文关键字" + "正文" : [""] * len(content) + } + ] - + + > list() + +""" + +openaikey_list = ["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", + "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", + "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", + "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", + "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", + "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", + "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", + "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", + "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"] + + +redis_key_name_openaikey_list = "openaikey_list_{}".format(str(localhostip)) + +redis_title = "redis_title" + +redis_title_ing = "redis_title_ing" + +redis_small_task = "redis_small_task" + +redis_res = "redis_res" + +for i in openaikey_list: + redis_.rpush(redis_key_name_openaikey_list, i) + + +def chat_kaitibaogao(api_key, uuid, main_parameter): + # t = Thread(target=chat_kaitibaogao, args=(api_key, + # uuid, + # main_parameter + # time.sleep(1) + openai.api_key = api_key + res = openai.ChatCompletion.create( + model="gpt-3.5-turbo", + messages=[ + {"role": "user", "content": kaitibaogao_prompt.format(main_parameter[0])}, + ], + temperature=0.5 + ) + kaitibaogao = res.choices[0].message.content + # kaitibaogao_path = os.path.join(, "kaitibaogao.txt") + # with open(kaitibaogao_path, 'w', encoding='utf8') as f_kaitibaogao: + # f_kaitibaogao.write(kaitibaogao) + + redis_.rpush(redis_key_name_openaikey_list, api_key) + lock.acquire() + res_dict_str = redis_.hget(redis_res, uuid) + res_dict = json.loads(res_dict_str) + res_dict["tasking_num"] += 1 + res_dict["开题报告"] = kaitibaogao + res_dict_str = json.dumps(res_dict, ensure_ascii=False) + redis_.hset(redis_res, uuid, res_dict_str) + lock.release() + + +def chat_abstract_keyword(api_key, uuid, main_parameter): + # api_key, + # uuid, + # main_parameter + # time.sleep(7) + openai.api_key = api_key + # 生成中文摘要 + res = openai.ChatCompletion.create( + model="gpt-3.5-turbo", + messages=[ + {"role": "user", "content": chinese_abstract_prompt.format(main_parameter[0])}, + ], + temperature=0.5 + ) + chinese_abstract = res.choices[0].message.content + # 生成英文的摘要 + res = openai.ChatCompletion.create( + model="gpt-3.5-turbo", + messages=[ + {"role": "user", "content": english_abstract_prompt.format(chinese_abstract)}, + ], + temperature=0.5 + ) + english_abstract = res.choices[0].message.content + # 生成中文关键字 + res = openai.ChatCompletion.create( + model="gpt-3.5-turbo", + messages=[ + {"role": "user", "content": chinese_keyword_prompt.format(chinese_abstract)}, + ], + temperature=0.5 + ) + chinese_keyword = res.choices[0].message.content + # 生成英文关键字 + res = openai.ChatCompletion.create( + model="gpt-3.5-turbo", + messages=[ + {"role": "user", "content": english_keyword_prompt.format(chinese_keyword)}, + ], + temperature=0.5 + ) + + english_keyword = res.choices[0].message.content + + paper_abstract_keyword = { + "中文摘要": chinese_abstract, + "英文摘要": english_abstract, + "中文关键词": chinese_keyword, + "英文关键词": english_keyword + } + + # json_str = json.dumps(paper_abstract_keyword, indent=4, ensure_ascii=False) + # abstract_keyword_path = os.path.join(uuid_path, "abstract_keyword.json") + # with open(abstract_keyword_path, 'w') as json_file: + # json_file.write(json_str) + # + # lock.acquire() + # api_key_list.append(api_key) + # lock.release() + redis_.rpush(redis_key_name_openaikey_list, api_key) + lock.acquire() + res_dict_str = redis_.hget(redis_res, uuid) + res_dict = json.loads(res_dict_str) + res_dict["tasking_num"] += 1 + res_dict["中文摘要"] = paper_abstract_keyword["中文摘要"] + res_dict["英文摘要"] = paper_abstract_keyword["英文摘要"] + res_dict["中文关键词"] = paper_abstract_keyword["中文关键词"] + res_dict["英文关键词"] = paper_abstract_keyword["英文关键词"] + res_dict_str = json.dumps(res_dict, ensure_ascii=False) + redis_.hset(redis_res, uuid, res_dict_str) + lock.release() + + +def chat_content(api_key, uuid, main_parameter): + # api_key, + # uuid, + # main_parameter : + # content_index, + # title, + # mulu, + # subtitle, + # prompt + # time.sleep(5) + + content_index = main_parameter[0] + title = main_parameter[1] + mulu = main_parameter[2] + subtitle = main_parameter[3] + prompt = main_parameter[4] + + res_content = "" + if subtitle[:2] == "@@": + res_content = subtitle[2:] + else: + openai.api_key = api_key + res = openai.ChatCompletion.create( + model="gpt-3.5-turbo", + messages=[ + {"role": "user", "content": prompt.format(title, mulu, subtitle)}, + ], + temperature=0.5 + ) + res_content = res.choices[0].message.content + redis_.rpush(redis_key_name_openaikey_list, api_key) + lock.acquire() + res_dict_str = redis_.hget(redis_res, uuid) + res_dict = json.loads(res_dict_str) + res_dict["tasking_num"] += 1 + table_of_contents = res_dict["table_of_contents"] + table_of_contents[content_index] = res_content + res_dict["table_of_contents"] = table_of_contents + res_dict_str = json.dumps(res_dict, ensure_ascii=False) + redis_.hset(redis_res, uuid, res_dict_str) + lock.release() + + +def threading_mulu(key_api, title, uuid): + ''' + 生成目录并吧任务拆解进入子任务的redis_list中和储存结果的redis_list中 + :return: + ''' + + # time.sleep(5) + # return [str(i) for i in range(20)] + + openai.api_key = key_api + res = openai.ChatCompletion.create( + model="gpt-3.5-turbo", + messages=[ + {"role": "user", "content": mulu_prompt.format(title)}, + ], + temperature=0.5 + ) + + redis_.rpush(redis_key_name_openaikey_list, key_api) + mulu = res.choices[0].message.content + mulu_list = str(mulu).split("\n") + mulu_list = [i.strip() for i in mulu_list if i != ""] + + print(mulu_list) + + cun_bool = False + table_of_contents = [mulu_list[0]] + + for i in mulu_list[1:]: + result_second_biaoti_list = re.findall(pantten_second_biaoti, i) + result_other_biaoti_list = re.findall(pantten_other_biaoti, i) + if result_second_biaoti_list != []: + table_of_contents.append("@@" + i) + cun_bool = True + continue + if cun_bool == False: + continue + else: + if result_other_biaoti_list != []: + table_of_contents.append("@@" + i) + else: + table_of_contents.append(i) + + print(table_of_contents) + # table_of_contents = table_of_contents[:3] + table_of_contents[-1:] + # print(table_of_contents) + + thanks_references_bool_table = table_of_contents[-3:] + + # thanks = "致谢" + # references = "参考文献" + if references in thanks_references_bool_table: + table_of_contents.remove(references) + + if thanks in thanks_references_bool_table: + table_of_contents.remove(thanks) + + table_of_contents.append(thanks) + table_of_contents.append(references) + + # if thanks not in thanks_bool_table: + # table_of_contents.insert(-1, "致谢") + # + # if thanks not in thanks_bool_table: + # table_of_contents.insert(-1, "致谢") + + print(len(table_of_contents)) + + small_task_list = [] + # api_key, + # index, + # title, + # mulu, + # subtitle, + # prompt + kaitibaogao_task = { + "task_type": "kaitibaogao", + "uuid": uuid, + "main_parameter": [title] + } + + chat_abstract_task = { + "task_type": "chat_abstract", + "uuid": uuid, + "main_parameter": [title] + } + small_task_list.append(kaitibaogao_task) + small_task_list.append(chat_abstract_task) + content_index = 0 + while True: + if content_index == len(table_of_contents): + break + subtitle = table_of_contents[content_index] + if content_index == 0: + prompt = first_title_prompt + elif subtitle == "参考文献": + prompt = references_prompt + elif subtitle == "致谢": + prompt = thank_prompt + else: + prompt = small_title_prompt + print("请求的所有参数", + content_index, + title, + subtitle, + prompt) + + paper_content = { + "task_type": "paper_content", + "uuid": uuid, + "main_parameter": [ + content_index, + title, + mulu, + subtitle, + prompt + ] + } + + small_task_list.append(paper_content) + content_index += 1 + + for small_task in small_task_list: + small_task = json.dumps(small_task, ensure_ascii=False) + redis_.rpush(redis_small_task, small_task) + + res = { + "uuid": uuid, + "num_small_task": len(small_task_list), + "tasking_num": 0, + "标题": title, + "目录": mulu, + "开题报告": "", + "任务书": "", + "中文摘要": "", + "英文摘要": "", + "中文关键词": "", + "英文关键词": "", + "正文": "", + "致谢": "", + "参考文献": "", + "table_of_contents": [""] * len(table_of_contents) + } + + res = json.dumps(res, ensure_ascii=False) + redis_.hset(redis_res, uuid, res) + + +def threading_1(): + # title, redis_key_name_openaikey_list + ''' + 生成目录 + :param title: + :param redis_key_name_openaikey_list: + :return: + ''' + while True: + if redis_.llen(redis_small_task) != 0: # 若队列中有元素就跳过 + time.sleep(1) + continue + elif redis_.llen(redis_title) != 0 and redis_.llen(redis_key_name_openaikey_list) != 0: + title_uuid_dict_str = redis_.lpop(redis_title).decode('UTF-8') + api_key = redis_.lpop(redis_key_name_openaikey_list).decode('UTF-8') + # redis_title:{"id": id_, "title": title} + title_uuid_dict = json.loads(title_uuid_dict_str) + + title = title_uuid_dict["title"] + uuid_id = title_uuid_dict["id"] + + t = Thread(target=threading_mulu, args=(api_key, + title, + uuid_id, + )) + t.start() + else: + time.sleep(1) + continue + + +def threading_2(): + ''' + 顺序读取子任务 + :return: + ''' + while True: + if redis_.llen(redis_small_task) != 0 and redis_.llen(redis_key_name_openaikey_list) != 0: + # 执行小标题的任务 + api_key = redis_.lpop(redis_key_name_openaikey_list).decode('UTF-8') + small_title = redis_.lpop(redis_small_task).decode('UTF-8') + small_title = json.loads(small_title) + task_type = small_title["task_type"] + uuid = small_title["uuid"] + main_parameter = small_title["main_parameter"] + + # "task_type": "paper_content", + # "uuid": uuid, + # "main_parameter": [ + # "task_type": "paper_content", + # "task_type": "chat_abstract", + # "task_type": "kaitibaogao", + + if task_type == "kaitibaogao": + t = Thread(target=chat_kaitibaogao, args=(api_key, + uuid, + main_parameter + )) + t.start() + elif task_type == "chat_abstract": + t = Thread(target=chat_abstract_keyword, args=(api_key, + uuid, + main_parameter + )) + t.start() + elif task_type == "paper_content": + t = Thread(target=chat_content, args=(api_key, + uuid, + main_parameter + )) + t.start() + + else: + time.sleep(1) + continue + + +def threading_3(): + while True: + res_end_list = [] + res_dict = redis_.hgetall(redis_res) + for key, values in res_dict.items(): + values_dict = json.loads(values) + # "num_small_task": len(small_task_list) - 1, + # "tasking_num": 0, + print(values_dict["num_small_task"]) + print(values_dict["tasking_num"]) + if int(values_dict["num_small_task"]) == int(values_dict["tasking_num"]): + res_end_list.append(key) + for key in res_end_list: + redis_.hdel(redis_res, key) + print(res_dict[key].decode("utf-8")) + res_str = res_dict[key].decode("utf-8") + json_str = json.dumps(res_str, indent=4, ensure_ascii=False) + print(project_data_txt_path) + print(key) + key = str(key, encoding="utf-8") + uuid_path = os.path.join(project_data_txt_path, key) + + print("uuid", key) + os.makedirs(uuid_path) + print("uuid_path", os.path.exists(uuid_path)) + paper_content_path = os.path.join(uuid_path, "paper_content.json") + with open(paper_content_path, 'w') as json_file: + json_file.write(json_str) + + +def main(title): + # print(request.remote_addr) + # title = request.json["title"] + + id_ = str(uuid.uuid1()) + print(id_) + redis_.rpush(redis_title, json.dumps({"id": id_, "title": title})) # 加入redis + # return_text = {"texts": {'id': id_, }, "probabilities": None, "status_code": 200} + + # threading_1 # 根据标题获取子任务,存入子任务序列 + # threading_2 # 根据子任务生成结果,存入结果序列 + # threading_3 # 根据存储的结果序列,看是否完成,如果完成输出json文件以及word + t = Thread(target=threading_1) + t.start() + t = Thread(target=threading_2) + t.start() + t = Thread(target=threading_3) + t.start() + + +if __name__ == '__main__': + main("大型商业建筑人员疏散设计研究")