From 53672a0582cdf0d7ae9d46da0d8ce804d0c3ab7c Mon Sep 17 00:00:00 2001
From: "majiahui@haimaqingfan.com" <majiahui@haimaqingfan.com>
Date: Wed, 12 Mar 2025 17:28:00 +0800
Subject: [PATCH] =?UTF-8?q?=E9=A6=96=E6=AC=A1=E6=8F=90=E4=BA=A4?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 README.md              |   0
 main.py                | 207 +++++++++++++++++++++++++++++++++
 model_api.py           | 179 +++++++++++++++++++++++++++++
 model_api_deepspeed.py | 169 +++++++++++++++++++++++++++
 model_api_vllm.py      | 303 +++++++++++++++++++++++++++++++++++++++++++++++++
 5 files changed, 858 insertions(+)
 create mode 100644 README.md
 create mode 100644 main.py
 create mode 100644 model_api.py
 create mode 100644 model_api_deepspeed.py
 create mode 100644 model_api_vllm.py

diff --git a/README.md b/README.md
new file mode 100644
index 0000000..e69de29
diff --git a/main.py b/main.py
new file mode 100644
index 0000000..46cee45
--- /dev/null
+++ b/main.py
@@ -0,0 +1,207 @@
+# 这是一个示例 Python 脚本。
+
+# 按 Shift+F10 执行或将其替换为您的代码。
+# 按 双击 Shift 在所有地方搜索类、文件、工具窗口、操作和设置。
+import faiss
+import numpy as np
+from tqdm import tqdm
+from sentence_transformers import SentenceTransformer
+import requests
+import time
+from flask import Flask, jsonify
+from flask import request
+
+app = Flask(__name__)
+app.config["JSON_AS_ASCII"] = False
+
+model = SentenceTransformer('/home/majiahui/project/models-llm/bge-large-zh-v1.5')
+propmt_connect = '''我是一名中医，你是一个中医的医生的助理，我的患者有一个症状，症状如下:
+{}
+根据这些症状，我通过查找资料，{}
+请根据上面的这些资料和方子，根据患者的症状帮我开出正确的药方和治疗方案'''
+
+propmt_connect_ziliao = '''在“{}”资料中，有如下相关内容：
+{}'''
+
+
+def dialog_line_parse(url, text):
+    """
+    将数据输入模型进行分析并输出结果
+    :param url: 模型url
+    :param text: 进入模型的数据
+    :return: 模型返回结果
+    """
+
+    response = requests.post(
+        url,
+        json=text,
+        timeout=100000
+    )
+    if response.status_code == 200:
+        return response.json()
+    else:
+        # logger.error(
+        #     "【{}】 Failed to get a proper response from remote "
+        #     "server. Status Code: {}. Response: {}"
+        #     "".format(url, response.status_code, response.text)
+        # )
+        print("【{}】 Failed to get a proper response from remote "
+              "server. Status Code: {}. Response: {}"
+              "".format(url, response.status_code, response.text))
+        return {}
+
+
+def shengcehng_array(data):
+    embs = model.encode(data, normalize_embeddings=True)
+    return embs
+
+def Building_vector_database(type, name, data):
+    data_ndarray = np.empty((0, 1024))
+    for sen in data:
+        data_ndarray = np.concatenate((data_ndarray, shengcehng_array([sen])))
+        print("data_ndarray.shape", data_ndarray.shape)
+
+    print("data_ndarray.shape", data_ndarray.shape)
+    np.save(f'data_np/{name}.npy', data_ndarray)
+
+
+
+def ulit_request_file(file, title):
+    file_name = file.filename
+    file_name_save = "data_file/{}.txt".format(title)
+    file.save(file_name_save)
+
+    try:
+        with open(file_name_save, encoding="gbk") as f:
+            content = f.read()
+    except:
+        with open(file_name_save, encoding="utf-8") as f:
+            content = f.read()
+    # elif file_name.split(".")[-1] == "docx":
+    #     content = docx2txt.process(file_name_save)
+
+    content_list = [i for i in content.split("\n")]
+
+    return content_list
+
+
+def main(question, db_type, top):
+    db_dict = {
+        "1": "yetianshi"
+    }
+    '''
+    定义文件路径
+    '''
+
+    '''
+    加载文件
+    '''
+
+    '''
+    文本分割
+    '''
+
+    '''
+    构建向量数据库
+        1. 正常匹配
+        2. 把文本使用大模型生成一个问题之后再进行匹配      
+    '''
+
+    '''
+    根据提问匹配上下文
+    '''
+    d = 1024
+    db_type_list = db_type.split(",")
+
+    paper_list_str = ""
+    for i in db_type_list:
+        embs = shengcehng_array([question])
+        index = faiss.IndexFlatL2(d)  # buid the index
+        data_np = np.load(f"data_np/{i}.npy")
+        data_str = open(f"data_file/{i}.txt").read().split("\n")
+        index.add(data_np)
+        D, I = index.search(embs, int(top))
+        print(I)
+
+        reference_list = []
+        for i in I[0]:
+            reference_list.append(data_str[i])
+
+        for i,j in enumerate(reference_list):
+            paper_list_str += "第{}篇\n{}\n".format(str(i+1), j)
+
+    '''
+    构造prompt
+    '''
+    print("paper_list_str", paper_list_str)
+    propmt_connect_ziliao_input = []
+    for i in db_type_list:
+        propmt_connect_ziliao_input.append(propmt_connect_ziliao.format(i, paper_list_str))
+
+    propmt_connect_ziliao_input_str = "，".join(propmt_connect_ziliao_input)
+    propmt_connect_input = propmt_connect.format(question, propmt_connect_ziliao_input_str)
+    '''
+    生成回答
+    '''
+    url_predict = "http://192.168.31.74:26000/predict"
+    url_search = "http://192.168.31.74:26000/search"
+
+    # data = {
+    #     "content": propmt_connect_input
+    # }
+    data = {
+        "content": propmt_connect_input,
+        "model": "qwq-32",
+        "top_p": 0.9,
+        "temperature": 0.6
+    }
+
+    res = dialog_line_parse(url_predict, data)
+    id_ = res["texts"]["id"]
+
+    data = {
+        "id": id_
+    }
+
+    while True:
+        res = dialog_line_parse(url_search, data)
+        if res["code"] == 200:
+            break
+        else:
+            time.sleep(1)
+    spilt_str = "</think>"
+    think, response = str(res["text"]).split(spilt_str)
+    return think, response
+
+
+@app.route("/upload_file", methods=["POST"])
+def upload_file():
+    print(request.remote_addr)
+    file = request.files.get('file')
+    title = request.form.get("title")
+    data = ulit_request_file(file, title)
+    Building_vector_database("1", title, data)
+    return_json = {
+        "code": 200,
+        "info": "上传完成"
+    }
+    return jsonify(return_json)  # 返回结果
+
+
+@app.route("/search", methods=["POST"])
+def search():
+    print(request.remote_addr)
+    texts = request.json["texts"]
+    text_type = request.json["text_type"]
+    top = request.json["top"]
+    think, response = main(texts, text_type, top)
+    return_json = {
+        "code": 200,
+        "think": think,
+        "response": response
+    }
+    return jsonify(return_json)  # 返回结果
+
+
+if __name__ == "__main__":
+    app.run(host="0.0.0.0", port=27000, threaded=True, debug=False)
diff --git a/model_api.py b/model_api.py
new file mode 100644
index 0000000..ed13ca7
--- /dev/null
+++ b/model_api.py
@@ -0,0 +1,179 @@
+import os
+os.environ["CUDA_VISIBLE_DEVICES"] = "1,2,3,4"
+from transformers import AutoModelForCausalLM, AutoTokenizer
+import logging
+from threading import Thread
+import requests
+import redis
+import uuid
+import time
+import json
+from flask import Flask, jsonify
+from flask import request
+
+app = Flask(__name__)
+app.config["JSON_AS_ASCII"] = False
+pool = redis.ConnectionPool(host='localhost', port=63179, max_connections=50,db=17, password="zhicheng123*")
+redis_ = redis.Redis(connection_pool=pool, decode_responses=True)
+
+# model config
+model_name = "/home/majiahui/project/models-llm/QwQ-32"
+model = AutoModelForCausalLM.from_pretrained(
+    model_name,
+    torch_dtype="auto",
+    device_map="auto"
+)
+tokenizer = AutoTokenizer.from_pretrained(model_name)
+
+db_key_query = 'query'
+db_key_querying = 'querying'
+db_key_result = 'result'
+batch_size = 15
+
+
+def main(prompt):
+    # prompt = "电视剧《人世间》导演和演员是谁"
+    messages = [
+        {"role": "user", "content": prompt}
+    ]
+    text = tokenizer.apply_chat_template(
+        messages,
+        tokenize=False,
+        add_generation_prompt=True
+    )
+
+    model_inputs = tokenizer([text], return_tensors="pt").to(model.device)
+
+    generated_ids = model.generate(
+        **model_inputs,
+        max_new_tokens=32768
+    )
+    generated_ids = [
+        output_ids[len(input_ids):] for input_ids, output_ids in zip(model_inputs.input_ids, generated_ids)
+    ]
+
+    response = tokenizer.batch_decode(generated_ids, skip_special_tokens=True)[0]
+    response = str(response).split("</think>")[1]
+    return response
+
+
+def classify():  # 调用模型，设置最大batch_size
+    while True:
+        if redis_.llen(db_key_query) == 0:  # 若队列中没有元素就继续获取
+            time.sleep(2)
+            continue
+
+        query = redis_.lpop(db_key_query).decode('UTF-8')  # 获取query的text
+        data_dict_path = json.loads(query)
+        path = data_dict_path['path']
+
+        with open(path, encoding='utf8') as f1:
+            # 加载文件的对象
+            data_dict = json.load(f1)
+
+        query_id = data_dict['id']
+        input_text = data_dict["text"]
+
+        output_text = main(input_text)
+
+        return_text = {
+            "input": input_text,
+            "output": output_text
+        }
+
+        load_result_path = "./new_data_logs/{}.json".format(query_id)
+        print("query_id: ", query_id)
+        print("load_result_path: ", load_result_path)
+
+        with open(load_result_path, 'w', encoding='utf8') as f2:
+            # ensure_ascii=False才能输入中文，否则是Unicode字符
+            # indent=2 JSON数据的缩进，美观
+            json.dump(return_text, f2, ensure_ascii=False, indent=4)
+        redis_.set(query_id, load_result_path, 86400)
+        # log.log('start at',
+        #         'query_id:{},load_result_path:{},return_text:{}, debug_id_1:{}, debug_id_2:{}, debug_id_3:{}'.format(
+        #             query_id, load_result_path, return_text)
+
+
+@app.route("/predict", methods=["POST"])
+def predict():
+    print(request.remote_addr)
+    content = request.json["content"]
+
+    id_ = str(uuid.uuid1())  # 为query生成唯一标识
+    print("uuid: ", id_)
+    d = {'id': id_, 'text': content}  # 绑定文本和query id
+
+    load_request_path = './request_data_logs/{}.json'.format(id_)
+    with open(load_request_path, 'w', encoding='utf8') as f2:
+        # ensure_ascii=False才能输入中文，否则是Unicode字符
+        # indent=2 JSON数据的缩进，美观
+        json.dump(d, f2, ensure_ascii=False, indent=4)
+    redis_.rpush(db_key_query, json.dumps({"id": id_, "path": load_request_path}))  # 加入redis
+    redis_.sadd(db_key_querying, id_)
+    return_text = {"texts": {'id': id_, }, "probabilities": None, "status_code": 200}
+    print("ok")
+    return jsonify(return_text)  # 返回结果
+
+
+@app.route("/search", methods=["POST"])
+def search():
+    id_ = request.json['id']  # 获取用户query中的文本 例如"I love you"
+    result = redis_.get(id_)  # 获取该query的模型结果
+    if result is not None:
+        # redis_.delete(id_)
+        result_path = result.decode('UTF-8')
+        with open(result_path, encoding='utf8') as f1:
+            # 加载文件的对象
+            result_dict = json.load(f1)
+        texts = result_dict["output"]
+        result_text = {'code': 200, 'text': texts, 'probabilities': None}
+    else:
+        querying_list = list(redis_.smembers("querying"))
+        querying_set = set()
+        for i in querying_list:
+            querying_set.add(i.decode())
+
+        querying_bool = False
+        if id_ in querying_set:
+            querying_bool = True
+
+        query_list_json = redis_.lrange(db_key_query, 0, -1)
+        query_set_ids = set()
+        for i in query_list_json:
+            data_dict = json.loads(i)
+            query_id = data_dict['id']
+            query_set_ids.add(query_id)
+
+        query_bool = False
+        if id_ in query_set_ids:
+            query_bool = True
+
+        if querying_bool == True and query_bool == True:
+            result_text = {'code': "201", 'text': "", 'probabilities': None}
+        elif querying_bool == True and query_bool == False:
+            result_text = {'code': "202", 'text': "", 'probabilities': None}
+        else:
+            result_text = {'code': "203", 'text': "", 'probabilities': None}
+            load_request_path = './request_data_logs_203/{}.json'.format(id_)
+            with open(load_request_path, 'w', encoding='utf8') as f2:
+                # ensure_ascii=False才能输入中文，否则是Unicode字符
+                # indent=2 JSON数据的缩进，美观
+                json.dump(result_text, f2, ensure_ascii=False, indent=4)
+
+    return jsonify(result_text)  # 返回结果
+
+
+t = Thread(target=classify)
+t.start()
+
+if __name__ == "__main__":
+    logging.basicConfig(level=logging.DEBUG,  # 控制台打印的日志级别
+                        filename='rewrite.log',
+                        filemode='a',  ##模式，有w和a，w就是写模式，每次都会重新写日志，覆盖之前的日志
+                        # a是追加模式，默认如果不写的话，就是追加模式
+                        format=
+                        '%(asctime)s - %(pathname)s[line:%(lineno)d] - %(levelname)s: %(message)s'
+                        # 日志格式
+                        )
+    app.run(host="0.0.0.0", port=26000, threaded=True, debug=False)
diff --git a/model_api_deepspeed.py b/model_api_deepspeed.py
new file mode 100644
index 0000000..54589ba
--- /dev/null
+++ b/model_api_deepspeed.py
@@ -0,0 +1,169 @@
+import os
+os.environ["CUDA_VISIBLE_DEVICES"] = "1,2,3,4"
+from transformers import AutoModelForCausalLM, AutoTokenizer
+import logging
+from threading import Thread
+import requests
+import redis
+import uuid
+import time
+import json
+from flask import Flask, jsonify
+from flask import request
+import deepspeed
+
+app = Flask(__name__)
+app.config["JSON_AS_ASCII"] = False
+pool = redis.ConnectionPool(host='localhost', port=63179, max_connections=50,db=17, password="zhicheng123*")
+redis_ = redis.Redis(connection_pool=pool, decode_responses=True)
+
+# model config
+model_name = "/home/majiahui/project/models-llm/QwQ-32"
+
+ds_config = {
+    "dtype": "fp16",
+    "tensor_parallel": {
+        "tp_size": 4
+    }
+}
+
+model = AutoModelForCausalLM.from_pretrained(model_name)
+tokenizer = AutoTokenizer.from_pretrained(model_name)
+model = deepspeed.init_inference(model, config=ds_config)
+
+db_key_query = 'query'
+db_key_querying = 'querying'
+db_key_result = 'result'
+batch_size = 15
+
+
+def main(prompt):
+    # prompt = "电视剧《人世间》导演和演员是谁"
+    inputs = tokenizer(prompt, return_tensors="pt").to("cuda")
+    output = model.generate(**inputs)
+    response = tokenizer.decode(output[0])
+    return response
+
+
+def classify():  # 调用模型，设置最大batch_size
+    while True:
+        if redis_.llen(db_key_query) == 0:  # 若队列中没有元素就继续获取
+            time.sleep(2)
+            continue
+
+        query = redis_.lpop(db_key_query).decode('UTF-8')  # 获取query的text
+        data_dict_path = json.loads(query)
+        path = data_dict_path['path']
+
+        with open(path, encoding='utf8') as f1:
+            # 加载文件的对象
+            data_dict = json.load(f1)
+
+        query_id = data_dict['id']
+        input_text = data_dict["text"]
+
+        output_text = main(input_text)
+
+        return_text = {
+            "input": input_text,
+            "output": output_text
+        }
+
+        load_result_path = "./new_data_logs/{}.json".format(query_id)
+        print("query_id: ", query_id)
+        print("load_result_path: ", load_result_path)
+
+        with open(load_result_path, 'w', encoding='utf8') as f2:
+            # ensure_ascii=False才能输入中文，否则是Unicode字符
+            # indent=2 JSON数据的缩进，美观
+            json.dump(return_text, f2, ensure_ascii=False, indent=4)
+        redis_.set(query_id, load_result_path, 86400)
+        # log.log('start at',
+        #         'query_id:{},load_result_path:{},return_text:{}, debug_id_1:{}, debug_id_2:{}, debug_id_3:{}'.format(
+        #             query_id, load_result_path, return_text)
+
+
+@app.route("/predict", methods=["POST"])
+def predict():
+    print(request.remote_addr)
+    content = request.json["content"]
+
+    id_ = str(uuid.uuid1())  # 为query生成唯一标识
+    print("uuid: ", id_)
+    d = {'id': id_, 'text': content}  # 绑定文本和query id
+
+    load_request_path = './request_data_logs/{}.json'.format(id_)
+    with open(load_request_path, 'w', encoding='utf8') as f2:
+        # ensure_ascii=False才能输入中文，否则是Unicode字符
+        # indent=2 JSON数据的缩进，美观
+        json.dump(d, f2, ensure_ascii=False, indent=4)
+    redis_.rpush(db_key_query, json.dumps({"id": id_, "path": load_request_path}))  # 加入redis
+    redis_.sadd(db_key_querying, id_)
+    return_text = {"texts": {'id': id_, }, "probabilities": None, "status_code": 200}
+    print("ok")
+    return jsonify(return_text)  # 返回结果
+
+
+@app.route("/search", methods=["POST"])
+def search():
+    id_ = request.json['id']  # 获取用户query中的文本 例如"I love you"
+    result = redis_.get(id_)  # 获取该query的模型结果
+    if result is not None:
+        # redis_.delete(id_)
+        result_path = result.decode('UTF-8')
+        with open(result_path, encoding='utf8') as f1:
+            # 加载文件的对象
+            result_dict = json.load(f1)
+        texts = result_dict["output"]
+        result_text = {'code': 200, 'text': texts, 'probabilities': None}
+    else:
+        querying_list = list(redis_.smembers("querying"))
+        querying_set = set()
+        for i in querying_list:
+            querying_set.add(i.decode())
+
+        querying_bool = False
+        if id_ in querying_set:
+            querying_bool = True
+
+        query_list_json = redis_.lrange(db_key_query, 0, -1)
+        query_set_ids = set()
+        for i in query_list_json:
+            data_dict = json.loads(i)
+            query_id = data_dict['id']
+            query_set_ids.add(query_id)
+
+        query_bool = False
+        if id_ in query_set_ids:
+            query_bool = True
+
+        if querying_bool == True and query_bool == True:
+            result_text = {'code': "201", 'text': "", 'probabilities': None}
+        elif querying_bool == True and query_bool == False:
+            result_text = {'code': "202", 'text': "", 'probabilities': None}
+        else:
+            result_text = {'code': "203", 'text': "", 'probabilities': None}
+            load_request_path = './request_data_logs_203/{}.json'.format(id_)
+            with open(load_request_path, 'w', encoding='utf8') as f2:
+                # ensure_ascii=False才能输入中文，否则是Unicode字符
+                # indent=2 JSON数据的缩进，美观
+                json.dump(result_text, f2, ensure_ascii=False, indent=4)
+
+    return jsonify(result_text)  # 返回结果
+
+
+t = Thread(target=classify)
+t.start()
+
+if __name__ == "__main__":
+    logging.basicConfig(level=logging.DEBUG,  # 控制台打印的日志级别
+                        filename='rewrite.log',
+                        filemode='a',  ##模式，有w和a，w就是写模式，每次都会重新写日志，覆盖之前的日志
+                        # a是追加模式，默认如果不写的话，就是追加模式
+                        format=
+                        '%(asctime)s - %(pathname)s[line:%(lineno)d] - %(levelname)s: %(message)s'
+                        # 日志格式
+                        )
+    app.run(host="0.0.0.0", port=26000, threaded=True, debug=False)
+
+
diff --git a/model_api_vllm.py b/model_api_vllm.py
new file mode 100644
index 0000000..b4cbc41
--- /dev/null
+++ b/model_api_vllm.py
@@ -0,0 +1,303 @@
+import os
+os.environ["CUDA_VISIBLE_DEVICES"] = "1,2,3,4"
+import argparse
+from typing import List, Tuple
+from threading import Thread
+from vllm import EngineArgs, LLMEngine, RequestOutput, SamplingParams
+from transformers import AutoTokenizer
+# from vllm.utils import FlexibleArgumentParser
+from flask import Flask, jsonify
+from flask import request
+import redis
+import time
+import json
+import uuid
+
+
+# http接口服务
+# app = FastAPI()
+app = Flask(__name__)
+app.config["JSON_AS_ASCII"] = False
+
+pool = redis.ConnectionPool(host='localhost', port=63179, max_connections=50,db=17, password="zhicheng123*")
+redis_ = redis.Redis(connection_pool=pool, decode_responses=True)
+
+db_key_query = 'query'
+db_key_querying = 'querying'
+db_key_queryset = 'queryset'
+db_key_result = 'result'
+db_key_error = 'error'
+batch_size = 2
+tokenizer = AutoTokenizer.from_pretrained("/home/majiahui/project/models-llm/QwQ-32")
+
+
+def smtp_f(name):
+    # 在下面的代码行中使用断点来调试脚本。
+    import smtplib
+    from email.mime.text import MIMEText
+    from email.header import Header
+
+    sender = '838878981@qq.com'  # 发送邮箱
+    receivers = ['838878981@qq.com']  # 接收邮箱
+    auth_code = "jfqtutaiwrtdbcge"  # 授权码
+
+    message = MIMEText('基础大模型出现错误，紧急', 'plain', 'utf-8')
+    message['From'] = Header("Sender<%s>" % sender)  # 发送者
+    message['To'] = Header("Receiver<%s>" % receivers[0])  # 接收者
+
+    subject = name
+    message['Subject'] = Header(subject, 'utf-8')
+
+    try:
+        server = smtplib.SMTP_SSL('smtp.qq.com', 465)
+        server.login(sender, auth_code)
+        server.sendmail(sender, receivers, message.as_string())
+        print("邮件发送成功")
+        server.close()
+    except smtplib.SMTPException:
+        print("Error: 无法发送邮件")
+
+class log:
+    def __init__(self):
+        pass
+
+    def log(*args, **kwargs):
+        format = '%Y/%m/%d-%H:%M:%S'
+        format_h = '%Y-%m-%d'
+        value = time.localtime(int(time.time()))
+        dt = time.strftime(format, value)
+        dt_log_file = time.strftime(format_h, value)
+        log_file = 'log_file/access-%s' % dt_log_file + ".log"
+        if not os.path.exists(log_file):
+            with open(os.path.join(log_file), 'w', encoding='utf-8') as f:
+                print(dt, *args, file=f, **kwargs)
+        else:
+            with open(os.path.join(log_file), 'a+', encoding='utf-8') as f:
+                print(dt, *args, file=f, **kwargs)
+
+
+def initialize_engine() -> LLMEngine:
+    """Initialize the LLMEngine from the command line arguments."""
+    # model_dir = "/home/majiahui/project/models-llm/Qwen-0_5B-Chat"
+    # model_dir = "/home/majiahui/project/models-llm/openbuddy-qwen2.5llamaify-7b_train_11_prompt_mistral_gpt_xiaobiaot_real_paper"
+    # model_dir = "/home/majiahui/project/models-llm/openbuddy-qwen2.5llamaify-7b_train_11_prompt_mistral_gpt_xiaobiaot_real_paper_2"
+    # model_dir = "/home/majiahui/project/models-llm/Qwen2.5-7B-Instruct-1M"
+    # model_dir = "/home/majiahui/project/models-llm/openbuddy-qwen2.5llamaify-7b-v23.1-200k"
+    model_dir = "/home/majiahui/project/models-llm/QwQ-32"
+    args = EngineArgs(model_dir)
+    args.max_num_seqs = 2  # batch最大20条样本
+    # args.gpu_memory_utilization = 0.8
+    args.tensor_parallel_size = 4
+    args.max_model_len=8192
+    # 加载模型
+    return LLMEngine.from_engine_args(args)
+
+engine = initialize_engine()
+
+
+def create_test_prompts(prompt_texts, query_ids, sampling_params_list) -> List[Tuple[str,str, SamplingParams]]:
+    """Create a list of test prompts with their sampling parameters."""
+
+    return_list = []
+
+    for i,j,k in zip(prompt_texts, query_ids, sampling_params_list):
+        return_list.append((i, j, k))
+    return return_list
+
+
+def process_requests(engine: LLMEngine,
+                     test_prompts: List[Tuple[str, str, SamplingParams]]):
+    """Continuously process a list of prompts and handle the outputs."""
+
+    return_list = []
+    while test_prompts or engine.has_unfinished_requests():
+        if test_prompts:
+            prompt,  query_id, sampling_params = test_prompts.pop(0)
+            engine.add_request(str(query_id), prompt, sampling_params)
+
+        request_outputs: List[RequestOutput] = engine.step()
+
+        for request_output in request_outputs:
+            if request_output.finished:
+                return_list.append(request_output)
+    return return_list
+
+
+def main(prompt_texts, query_ids, sampling_params_list):
+    """Main function that sets up and runs the prompt processing."""
+
+    test_prompts = create_test_prompts(prompt_texts, query_ids, sampling_params_list)
+    return process_requests(engine, test_prompts)
+
+
+def classify(batch_size):  # 调用模型，设置最大batch_size
+    while True:
+        texts = []
+        query_ids = []
+        sampling_params_list = []
+        if redis_.llen(db_key_query) == 0:  # 若队列中没有元素就继续获取
+            time.sleep(2)
+            continue
+
+        # for i in range(min(redis_.llen(db_key_query), batch_size)):
+        while True:
+            query = redis_.lpop(db_key_query)  # 获取query的text
+            if query == None:
+                break
+
+            query = query.decode('UTF-8')
+            data_dict_path = json.loads(query)
+
+            path = data_dict_path['path']
+            with open(path, encoding='utf8') as f1:
+                # 加载文件的对象
+                data_dict = json.load(f1)
+            # query_ids.append(json.loads(query)['id'])
+            # texts.append(json.loads(query)['text'])  # 拼接若干text 为batch
+            query_id = data_dict['id']
+            print("query_id", query_id)
+            text = data_dict["text"]
+            model = data_dict["model"]
+            top_p = data_dict["top_p"]
+            temperature = data_dict["temperature"]
+            presence_penalty = 1.1
+            max_tokens = 8192
+            query_ids.append(query_id)
+            messages = [
+                {"role": "user", "content": text}
+            ]
+            text = tokenizer.apply_chat_template(
+                messages,
+                tokenize=False,
+                add_generation_prompt=True
+            )
+            texts.append(text)
+            # sampling_params = SamplingParams(temperature=0.3, top_p=0.5, stop="<|end|>", presence_penalty=1.1, max_tokens=8192)
+            sampling_params_list.append(SamplingParams(
+                temperature=temperature,
+                top_p=top_p,
+                stop="<|im_end|>",
+                presence_penalty=presence_penalty,
+                max_tokens=max_tokens
+            ))
+            if len(texts) == batch_size:
+                break
+
+        print("texts", len(texts))
+        print("query_ids", len(query_ids))
+        print("sampling_params_list", len(sampling_params_list))
+        outputs = main(texts, query_ids, sampling_params_list)
+
+        print("预测完成")
+        generated_text_dict = {}
+        print("outputs", len(outputs))
+        for i, output in enumerate(outputs):
+            index = output.request_id
+            print(index)
+            generated_text = output.outputs[0].text
+            generated_text_dict[index] = generated_text
+
+        print(generated_text_dict)
+        for id_, output in generated_text_dict.items():
+
+            return_text = {"texts": output, "probabilities": None, "status_code": 200}
+            load_result_path = "./new_data_logs/{}.json".format(id_)
+            with open(load_result_path, 'w', encoding='utf8') as f2:
+                # ensure_ascii=False才能输入中文，否则是Unicode字符
+                # indent=2 JSON数据的缩进，美观
+                json.dump(return_text, f2, ensure_ascii=False, indent=4)
+            redis_.set(id_, load_result_path, 86400)
+            # redis_.set(id_, load_result_path, 30)
+            redis_.srem(db_key_querying, id_)
+            log.log('start at',
+                    'query_id:{},load_result_path:{},return_text:{}'.format(
+                        id_, load_result_path, return_text))
+
+
+@app.route("/predict", methods=["POST"])
+def predict():
+    content = request.json["content"]  # 获取用户query中的文本 例如"I love you"
+    model = request.json["model"]
+    top_p = request.json["top_p"]
+    temperature = request.json["temperature"]
+    id_ = str(uuid.uuid1())  # 为query生成唯一标识
+    print("uuid: ", uuid)
+    d = {'id': id_, 'text': content, 'model': model, 'top_p': top_p,'temperature': temperature}  # 绑定文本和query id
+    print(d)
+    try:
+        load_request_path = './request_data_logs/{}.json'.format(id_)
+        with open(load_request_path, 'w', encoding='utf8') as f2:
+            # ensure_ascii=False才能输入中文，否则是Unicode字符
+            # indent=2 JSON数据的缩进，美观
+            json.dump(d, f2, ensure_ascii=False, indent=4)
+        redis_.rpush(db_key_query, json.dumps({"id": id_, "path": load_request_path}))  # 加入redis
+        redis_.sadd(db_key_querying, id_)
+        redis_.sadd(db_key_queryset, id_)
+        return_text = {"texts": {'id': id_, }, "probabilities": None, "status_code": 200}
+    except:
+        return_text = {"texts": {'id': id_, }, "probabilities": None, "status_code": 400}
+        smtp_f("vllm-main-paper")
+    return jsonify(return_text)  # 返回结果
+
+
+@app.route("/search", methods=["POST"])
+def search():
+    id_ = request.json['id']  # 获取用户query中的文本 例如"I love you"
+    result = redis_.get(id_)  # 获取该query的模型结果
+    try:
+        if result is not None:
+            result_path = result.decode('UTF-8')
+            with open(result_path, encoding='utf8') as f1:
+                # 加载文件的对象
+                result_dict = json.load(f1)
+            code = result_dict["status_code"]
+            texts = result_dict["texts"]
+            probabilities = result_dict["probabilities"]
+            if str(code) == 400:
+                redis_.rpush(db_key_error, json.dumps({"id": id_}))
+                return False
+            result_text = {'code': code, 'text': texts, 'probabilities': probabilities}
+        else:
+            querying_list = list(redis_.smembers(db_key_querying))
+            querying_set = set()
+            for i in querying_list:
+                querying_set.add(i.decode())
+
+            querying_bool = False
+            if id_ in querying_set:
+                querying_bool = True
+
+            query_list_json = redis_.lrange(db_key_query, 0, -1)
+            query_set_ids = set()
+            for i in query_list_json:
+                data_dict = json.loads(i)
+                query_id = data_dict['id']
+                query_set_ids.add(query_id)
+
+            query_bool = False
+            if id_ in query_set_ids:
+                query_bool = True
+
+            if querying_bool == True and query_bool == True:
+                result_text = {'code': "201", 'text': "", 'probabilities': None}
+            elif querying_bool == True and query_bool == False:
+                result_text = {'code': "202", 'text': "", 'probabilities': None}
+            else:
+                result_text = {'code': "203", 'text': "", 'probabilities': None}
+                load_request_path = './request_data_logs_203/{}.json'.format(id_)
+                with open(load_request_path, 'w', encoding='utf8') as f2:
+                    # ensure_ascii=False才能输入中文，否则是Unicode字符
+                    # indent=2 JSON数据的缩进，美观
+                    json.dump(result_text, f2, ensure_ascii=False, indent=4)
+    except:
+        smtp_f("vllm-main")
+        result_text = {'code': "400", 'text': "", 'probabilities': None}
+    return jsonify(result_text)  # 返回结果
+
+
+t = Thread(target=classify, args=(batch_size,))
+t.start()
+
+
+if __name__ == "__main__":
+    app.run(debug=False, host='0.0.0.0', port=26000)
\ No newline at end of file