from flask import Flask, jsonify, Response from flask import request import redis import uuid import json import time import threading from threading import Thread from flask import send_file, send_from_directory import os from flask import make_response import openai import base64 import re import urllib.parse as pa import socket s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) s.connect(("8.8.8.8", 80)) localhostip = s.getsockname()[0] lock = threading.RLock() pool = redis.ConnectionPool(host='localhost', port=6379, max_connections=50, db=2) redis_ = redis.Redis(connection_pool=pool, decode_responses=True) pantten_second_biaoti = '[2二ⅡⅠ][、.]\s{0,}?[\u4e00-\u9fa5]+' pantten_other_biaoti = '[2-9二三四五六七八九ⅡⅢⅣⅤⅥⅦⅧⅨ][、.]\s{0,}?[\u4e00-\u9fa5]+' mulu_prompt = "请帮我根据题目为“{}”生成一个论文目录" first_title_prompt = "论文题目是“{}”，目录是“{}”，请把其中的大标题“{}”的内容续写完整，保证续写内容不少于800字" small_title_prompt = "论文题目是“{}”，目录是“{}”，请把其中的小标题“{}”的内容续写完整，保证续写内容不少于800字" references_prompt = "论文题目是“{}”，目录是“{}”，请为这篇论文生成15篇中文的{}，要求其中有有中文参考文献不低于12篇，英文参考文献不低于2篇" thank_prompt = "论文题目是“{}”，目录是“{}”，请把其中的{}部分续写完整" kaitibaogao_prompt = "请以《{}》为题目生成研究的主要的内容、背景、目的、意义，要求不少于100字" chinese_abstract_prompt = "请以《{}》为题目生成论文摘要，要求不少于1500字" english_abstract_prompt = "请把“{}”这段文字翻译成英文" chinese_keyword_prompt = "请为“{}”这段论文摘要生成3-5个关键字" english_keyword_prompt = "请把“{}”这几个关键字翻译成英文" thanks = "致谢" references = "参考文献" dabiaoti = ["二", "三", "四", "五", "六", "七", "八", "九"] project_data_txt_path = "/home/majiahui/ChatGPT_Sever/new_data_txt" """ key_list = [ {"ip": key-api}, {"ip": key-api}, {"ip": key-api}, ] redis_title = [] redis_title_ing = [] redis_small_task = [ { uuid, api_key, mulu_title_id, title, mulu, subtitle, prompt } ] redis_res = [ { "uuid": "完成进度": "标题": "中文摘要":"", "英文摘要" "中文关键字" "英文关键字" "正文" : [""] * len(content) } ] - > list() """ openaikey_list = ["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", "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", "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", "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", "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", "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", "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", "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", "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"] redis_key_name_openaikey_list = "openaikey_list_{}".format(str(localhostip)) redis_title = "redis_title" redis_title_ing = "redis_title_ing" redis_small_task = "redis_small_task" redis_res = "redis_res" for i in openaikey_list: redis_.rpush(redis_key_name_openaikey_list, i) def chat_kaitibaogao(api_key, uuid, main_parameter): # t = Thread(target=chat_kaitibaogao, args=(api_key, # uuid, # main_parameter # time.sleep(1) openai.api_key = api_key res = openai.ChatCompletion.create( model="gpt-3.5-turbo", messages=[ {"role": "user", "content": kaitibaogao_prompt.format(main_parameter[0])}, ], temperature=0.5 ) kaitibaogao = res.choices[0].message.content # kaitibaogao_path = os.path.join(, "kaitibaogao.txt") # with open(kaitibaogao_path, 'w', encoding='utf8') as f_kaitibaogao: # f_kaitibaogao.write(kaitibaogao) redis_.rpush(redis_key_name_openaikey_list, api_key) lock.acquire() res_dict_str = redis_.hget(redis_res, uuid) res_dict = json.loads(res_dict_str) res_dict["tasking_num"] += 1 res_dict["开题报告"] = kaitibaogao res_dict_str = json.dumps(res_dict, ensure_ascii=False) redis_.hset(redis_res, uuid, res_dict_str) lock.release() def chat_abstract_keyword(api_key, uuid, main_parameter): # api_key, # uuid, # main_parameter # time.sleep(7) openai.api_key = api_key # 生成中文摘要 res = openai.ChatCompletion.create( model="gpt-3.5-turbo", messages=[ {"role": "user", "content": chinese_abstract_prompt.format(main_parameter[0])}, ], temperature=0.5 ) chinese_abstract = res.choices[0].message.content # 生成英文的摘要 res = openai.ChatCompletion.create( model="gpt-3.5-turbo", messages=[ {"role": "user", "content": english_abstract_prompt.format(chinese_abstract)}, ], temperature=0.5 ) english_abstract = res.choices[0].message.content # 生成中文关键字 res = openai.ChatCompletion.create( model="gpt-3.5-turbo", messages=[ {"role": "user", "content": chinese_keyword_prompt.format(chinese_abstract)}, ], temperature=0.5 ) chinese_keyword = res.choices[0].message.content # 生成英文关键字 res = openai.ChatCompletion.create( model="gpt-3.5-turbo", messages=[ {"role": "user", "content": english_keyword_prompt.format(chinese_keyword)}, ], temperature=0.5 ) english_keyword = res.choices[0].message.content paper_abstract_keyword = { "中文摘要": chinese_abstract, "英文摘要": english_abstract, "中文关键词": chinese_keyword, "英文关键词": english_keyword } # json_str = json.dumps(paper_abstract_keyword, indent=4, ensure_ascii=False) # abstract_keyword_path = os.path.join(uuid_path, "abstract_keyword.json") # with open(abstract_keyword_path, 'w') as json_file: # json_file.write(json_str) # # lock.acquire() # api_key_list.append(api_key) # lock.release() redis_.rpush(redis_key_name_openaikey_list, api_key) lock.acquire() res_dict_str = redis_.hget(redis_res, uuid) res_dict = json.loads(res_dict_str) res_dict["tasking_num"] += 1 res_dict["中文摘要"] = paper_abstract_keyword["中文摘要"] res_dict["英文摘要"] = paper_abstract_keyword["英文摘要"] res_dict["中文关键词"] = paper_abstract_keyword["中文关键词"] res_dict["英文关键词"] = paper_abstract_keyword["英文关键词"] res_dict_str = json.dumps(res_dict, ensure_ascii=False) redis_.hset(redis_res, uuid, res_dict_str) lock.release() def chat_content(api_key, uuid, main_parameter): # api_key, # uuid, # main_parameter ： # content_index, # title, # mulu, # subtitle, # prompt # time.sleep(5) content_index = main_parameter[0] title = main_parameter[1] mulu = main_parameter[2] subtitle = main_parameter[3] prompt = main_parameter[4] res_content = "" if subtitle[:2] == "@@": res_content = subtitle[2:] else: openai.api_key = api_key res = openai.ChatCompletion.create( model="gpt-3.5-turbo", messages=[ {"role": "user", "content": prompt.format(title, mulu, subtitle)}, ], temperature=0.5 ) res_content = res.choices[0].message.content redis_.rpush(redis_key_name_openaikey_list, api_key) lock.acquire() res_dict_str = redis_.hget(redis_res, uuid) res_dict = json.loads(res_dict_str) res_dict["tasking_num"] += 1 table_of_contents = res_dict["table_of_contents"] table_of_contents[content_index] = res_content res_dict["table_of_contents"] = table_of_contents res_dict_str = json.dumps(res_dict, ensure_ascii=False) redis_.hset(redis_res, uuid, res_dict_str) lock.release() def threading_mulu(key_api, title, uuid): ''' 生成目录并吧任务拆解进入子任务的redis_list中和储存结果的redis_list中 :return: ''' # time.sleep(5) # return [str(i) for i in range(20)] openai.api_key = key_api res = openai.ChatCompletion.create( model="gpt-3.5-turbo", messages=[ {"role": "user", "content": mulu_prompt.format(title)}, ], temperature=0.5 ) redis_.rpush(redis_key_name_openaikey_list, key_api) mulu = res.choices[0].message.content mulu_list = str(mulu).split("\n") mulu_list = [i.strip() for i in mulu_list if i != ""] print(mulu_list) cun_bool = False table_of_contents = [mulu_list[0]] for i in mulu_list[1:]: result_second_biaoti_list = re.findall(pantten_second_biaoti, i) result_other_biaoti_list = re.findall(pantten_other_biaoti, i) if result_second_biaoti_list != []: table_of_contents.append("@@" + i) cun_bool = True continue if cun_bool == False: continue else: if result_other_biaoti_list != []: table_of_contents.append("@@" + i) else: table_of_contents.append(i) print(table_of_contents) # table_of_contents = table_of_contents[:3] + table_of_contents[-1:] # print(table_of_contents) thanks_references_bool_table = table_of_contents[-3:] # thanks = "致谢" # references = "参考文献" if references in thanks_references_bool_table: table_of_contents.remove(references) if thanks in thanks_references_bool_table: table_of_contents.remove(thanks) table_of_contents.append(thanks) table_of_contents.append(references) # if thanks not in thanks_bool_table: # table_of_contents.insert(-1, "致谢") # # if thanks not in thanks_bool_table: # table_of_contents.insert(-1, "致谢") print(len(table_of_contents)) small_task_list = [] # api_key, # index, # title, # mulu, # subtitle, # prompt kaitibaogao_task = { "task_type": "kaitibaogao", "uuid": uuid, "main_parameter": [title] } chat_abstract_task = { "task_type": "chat_abstract", "uuid": uuid, "main_parameter": [title] } small_task_list.append(kaitibaogao_task) small_task_list.append(chat_abstract_task) content_index = 0 while True: if content_index == len(table_of_contents): break subtitle = table_of_contents[content_index] if content_index == 0: prompt = first_title_prompt elif subtitle == "参考文献": prompt = references_prompt elif subtitle == "致谢": prompt = thank_prompt else: prompt = small_title_prompt print("请求的所有参数", content_index, title, subtitle, prompt) paper_content = { "task_type": "paper_content", "uuid": uuid, "main_parameter": [ content_index, title, mulu, subtitle, prompt ] } small_task_list.append(paper_content) content_index += 1 for small_task in small_task_list: small_task = json.dumps(small_task, ensure_ascii=False) redis_.rpush(redis_small_task, small_task) res = { "uuid": uuid, "num_small_task": len(small_task_list), "tasking_num": 0, "标题": title, "目录": mulu, "开题报告": "", "任务书": "", "中文摘要": "", "英文摘要": "", "中文关键词": "", "英文关键词": "", "正文": "", "致谢": "", "参考文献": "", "table_of_contents": [""] * len(table_of_contents) } res = json.dumps(res, ensure_ascii=False) redis_.hset(redis_res, uuid, res) def threading_1(): # title, redis_key_name_openaikey_list ''' 生成目录 :param title: :param redis_key_name_openaikey_list: :return: ''' while True: if redis_.llen(redis_small_task) != 0: # 若队列中有元素就跳过 time.sleep(1) continue elif redis_.llen(redis_title) != 0 and redis_.llen(redis_key_name_openaikey_list) != 0: title_uuid_dict_str = redis_.lpop(redis_title).decode('UTF-8') api_key = redis_.lpop(redis_key_name_openaikey_list).decode('UTF-8') # redis_title:{"id": id_, "title": title} title_uuid_dict = json.loads(title_uuid_dict_str) title = title_uuid_dict["title"] uuid_id = title_uuid_dict["id"] t = Thread(target=threading_mulu, args=(api_key, title, uuid_id, )) t.start() else: time.sleep(1) continue def threading_2(): ''' 顺序读取子任务 :return: ''' while True: if redis_.llen(redis_small_task) != 0 and redis_.llen(redis_key_name_openaikey_list) != 0: # 执行小标题的任务 api_key = redis_.lpop(redis_key_name_openaikey_list).decode('UTF-8') small_title = redis_.lpop(redis_small_task).decode('UTF-8') small_title = json.loads(small_title) task_type = small_title["task_type"] uuid = small_title["uuid"] main_parameter = small_title["main_parameter"] # "task_type": "paper_content", # "uuid": uuid, # "main_parameter": [ # "task_type": "paper_content", # "task_type": "chat_abstract", # "task_type": "kaitibaogao", if task_type == "kaitibaogao": t = Thread(target=chat_kaitibaogao, args=(api_key, uuid, main_parameter )) t.start() elif task_type == "chat_abstract": t = Thread(target=chat_abstract_keyword, args=(api_key, uuid, main_parameter )) t.start() elif task_type == "paper_content": t = Thread(target=chat_content, args=(api_key, uuid, main_parameter )) t.start() else: time.sleep(1) continue def threading_3(): while True: res_end_list = [] res_dict = redis_.hgetall(redis_res) for key, values in res_dict.items(): values_dict = json.loads(values) # "num_small_task": len(small_task_list) - 1, # "tasking_num": 0, print(values_dict["num_small_task"]) print(values_dict["tasking_num"]) if int(values_dict["num_small_task"]) == int(values_dict["tasking_num"]): res_end_list.append(key) for key in res_end_list: redis_.hdel(redis_res, key) print(res_dict[key].decode("utf-8")) res_str = res_dict[key].decode("utf-8") json_str = json.dumps(res_str, indent=4, ensure_ascii=False) print(project_data_txt_path) print(key) key = str(key, encoding="utf-8") uuid_path = os.path.join(project_data_txt_path, key) print("uuid", key) os.makedirs(uuid_path) print("uuid_path", os.path.exists(uuid_path)) paper_content_path = os.path.join(uuid_path, "paper_content.json") with open(paper_content_path, 'w') as json_file: json_file.write(json_str) def main(title): # print(request.remote_addr) # title = request.json["title"] id_ = str(uuid.uuid1()) print(id_) redis_.rpush(redis_title, json.dumps({"id": id_, "title": title})) # 加入redis # return_text = {"texts": {'id': id_, }, "probabilities": None, "status_code": 200} # threading_1 # 根据标题获取子任务，存入子任务序列 # threading_2 # 根据子任务生成结果，存入结果序列 # threading_3 # 根据存储的结果序列，看是否完成，如果完成输出json文件以及word t = Thread(target=threading_1) t.start() t = Thread(target=threading_2) t.start() t = Thread(target=threading_3) t.start() if __name__ == '__main__': main("大型商业建筑人员疏散设计研究")