mulu_ner/generate_mulu.py

import time

from tqdm import tqdm
import random
import requests
import json
import threading
from threading import Thread
import redis

lock = threading.RLock()
pool = redis.ConnectionPool(host='104.244.90.248', port=63179, max_connections=50, db=10, password='Zhicheng123*')
redis_ = redis.Redis(connection_pool=pool, decode_responses=True)

with open("api_key.txt", "r",) as f:
    a = f.read()
    a = a.split("\n")

redis_key_name_openaikey_list = "openaikey_list"
redis_zirenwu = "redis_zirenwu"

api_key_list = []
for i in a:
    api_key_list.append(str(i.split("----")[-1]))

for i in api_key_list:
    redis_.rpush(redis_key_name_openaikey_list, i)

lock = threading.RLock()

prompt_dict = {
    "mulu_prompt": "为论文题目“{}”生成目录，要求只有一级标题和二级标题，一级标题使用中文数字 例如一、xxx；二级标题使用阿拉伯数字 例如1.1 xxx；一级标题不少于7个；每个一级标题至少包含3个二级标题",
    "beijing_prompt": "以“{}”为论文题目，写一段题目来源的背景，要求字数在200字以内",
    "zongjie_prompt": "以“{}”为论文题目，写一个论文简短总结，要求在300字以内",
    "zongshu_prompt": "请写出以《{}》为课题的国内外研究状况综述，字数在800字左右",
    "yanjiubeijingyiyi_prompt": "请分别写出以《{}》为课题的研究背景和意义，字数不少于1000字",
    "jianjie_prompt": "请帮我生成《{}》为题目的研究内容，包括整体简介和分最少三个方面总结"
}

with open("./data/题目3.txt", encoding="utf-8") as f:
    text = f.read()

text_list = text.split("\n")

title_list = []
for i in text_list:
    title_list.append(i.split("@@@@@")[0])

random.shuffle(title_list)

print(len(title_list))

zirenwu_list = []

for title in title_list:
    for prompt in prompt_dict:
        zirenwu_list.append((prompt, str(prompt_dict[prompt]).format(title)))

for i in zirenwu_list:
    redis_.rpush(redis_zirenwu, str(i))

# def request_api_chatgpt(api_key, prompt):
#     OPENAI_API_KEY = api_key
#     url = "https://api.openai.com/v1/chat/completions"
#     headers = {
#         "Content-Type": "application/json",
#         "Authorization": f"Bearer {OPENAI_API_KEY}"
#     }
#     data = {
#         "model": "gpt-3.5-turbo",
#         "messages": [
#             {"role": "user", "content": prompt},
#         ],
#         "temperature": 0.5
#     }
#     response = requests.post(url,
#                              headers=headers,
#                              data=json.dumps(data),
#                              timeout=240)
#     print("response", response)
#
#     return response
#
# def task(api_key, title):
#     try:
#         for pormpt_dan in prompt_dict:
#             name = pormpt_dan.split("_")[0]
#             print("pormpt_dan", pormpt_dan)
#             print("prompt_dict", prompt_dict)
#             prompt = str(prompt_dict[pormpt_dan]).format(title)
#             print("api_key", api_key)
#             print("prompt", prompt)
#             response = request_api_chatgpt(api_key, prompt)
#             res = response.json()
#             text = res["choices"][0]["message"]["content"]
#             lock.acquire()
#
#             with open("/home/majiahui/mulu_ner/data/paper_prompt_title_new/title_{}_data.txt".format(name), mode="a") as f:
#                 f.write(title)
#                 f.write("\n**********************************************\n")
#                 f.write(text)
#                 f.write("\n++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++\n")
#             lock.release()
#             time.sleep(2)
#         lock.acquire()
#         api_key_list.append(api_key)
#         lock.release()
#         print(1)
#     except:
#         print()
#         time.sleep(5)
#         lock.acquire()
#         api_key_list.append(api_key)
#         lock.release()
#         print(2)

def request_api_chatgpt(api_key, task_type, prompt):
    global api_key_list
    global zirenwu_list
    try:
        OPENAI_API_KEY = api_key
        url = "https://api.openai.com/v1/chat/completions"
        headers = {
            "Content-Type": "application/json",
            "Authorization": f"Bearer {OPENAI_API_KEY}"
        }
        data = {
            "model": "gpt-3.5-turbo",
            "messages": [
                {"role": "user", "content": prompt},
            ],
            "temperature": 0.5
        }
        response = requests.post(url,
                                 headers=headers,
                                 data=json.dumps(data),
                                 timeout=240)

        res = response.json()
        text = res["choices"][0]["message"]["content"]
        lock.acquire()
        # api_key_list.append(api_key)
        redis_.rpush(redis_key_name_openaikey_list, api_key)

        with open("/home/majiahui/mulu_ner/data/paper_prompt_title_3/title_{}_data.txt".format(task_type), mode="a") as f:
            f.write(prompt)
            f.write("**************")
            f.write(text)
            f.write("\n")
        lock.release()

    except:
        print("task_type_bad", task_type)
        print("api_key_bad", api_key)
        time.sleep(5)
        lock.acquire()
        redis_.rpush(redis_key_name_openaikey_list, api_key)
        redis_.rpush(redis_zirenwu, str((task_type, prompt)))
        lock.release()


if __name__ == '__main__':
    while True:
        if redis_.llen(redis_zirenwu) == 0:
            time.sleep(1)
            continue
        elif redis_.llen(redis_zirenwu) != 0 and redis_.llen(redis_key_name_openaikey_list) != 0:
            lock.acquire()
            api_key = redis_.lpop(redis_key_name_openaikey_list)
            api_key = api_key.decode('UTF-8')
            dan_zirenwu = redis_.lpop(redis_zirenwu)
            dan_zirenwu = dan_zirenwu.decode('UTF-8')
            lock.release()
            # dan_zirenwu = zirenwu_list.pop(0)
            dan_zirenwu = eval(dan_zirenwu)
            task_type, prompt = dan_zirenwu[0], dan_zirenwu[1]
            t = Thread(target=request_api_chatgpt, args=(api_key, task_type, prompt))
            t.start()
        elif redis_.llen(redis_key_name_openaikey_list) == 0:
            time.sleep(1)
            continue
        else:
            time.sleep(1)
            continue