majiahui@haimaqingfan.com
2 months ago
commit
53672a0582
5 changed files with 858 additions and 0 deletions
@ -0,0 +1,207 @@ |
|||
# 这是一个示例 Python 脚本。 |
|||
|
|||
# 按 Shift+F10 执行或将其替换为您的代码。 |
|||
# 按 双击 Shift 在所有地方搜索类、文件、工具窗口、操作和设置。 |
|||
import faiss |
|||
import numpy as np |
|||
from tqdm import tqdm |
|||
from sentence_transformers import SentenceTransformer |
|||
import requests |
|||
import time |
|||
from flask import Flask, jsonify |
|||
from flask import request |
|||
|
|||
app = Flask(__name__) |
|||
app.config["JSON_AS_ASCII"] = False |
|||
|
|||
model = SentenceTransformer('/home/majiahui/project/models-llm/bge-large-zh-v1.5') |
|||
propmt_connect = '''我是一名中医，你是一个中医的医生的助理，我的患者有一个症状，症状如下: |
|||
{} |
|||
根据这些症状，我通过查找资料，{} |
|||
请根据上面的这些资料和方子，根据患者的症状帮我开出正确的药方和治疗方案''' |
|||
|
|||
propmt_connect_ziliao = '''在“{}”资料中，有如下相关内容： |
|||
{}''' |
|||
|
|||
|
|||
def dialog_line_parse(url, text): |
|||
""" |
|||
将数据输入模型进行分析并输出结果 |
|||
:param url: 模型url |
|||
:param text: 进入模型的数据 |
|||
:return: 模型返回结果 |
|||
""" |
|||
|
|||
response = requests.post( |
|||
url, |
|||
json=text, |
|||
timeout=100000 |
|||
) |
|||
if response.status_code == 200: |
|||
return response.json() |
|||
else: |
|||
# logger.error( |
|||
# "【{}】 Failed to get a proper response from remote " |
|||
# "server. Status Code: {}. Response: {}" |
|||
# "".format(url, response.status_code, response.text) |
|||
# ) |
|||
print("【{}】 Failed to get a proper response from remote " |
|||
"server. Status Code: {}. Response: {}" |
|||
"".format(url, response.status_code, response.text)) |
|||
return {} |
|||
|
|||
|
|||
def shengcehng_array(data): |
|||
embs = model.encode(data, normalize_embeddings=True) |
|||
return embs |
|||
|
|||
def Building_vector_database(type, name, data): |
|||
data_ndarray = np.empty((0, 1024)) |
|||
for sen in data: |
|||
data_ndarray = np.concatenate((data_ndarray, shengcehng_array([sen]))) |
|||
print("data_ndarray.shape", data_ndarray.shape) |
|||
|
|||
print("data_ndarray.shape", data_ndarray.shape) |
|||
np.save(f'data_np/{name}.npy', data_ndarray) |
|||
|
|||
|
|||
|
|||
def ulit_request_file(file, title): |
|||
file_name = file.filename |
|||
file_name_save = "data_file/{}.txt".format(title) |
|||
file.save(file_name_save) |
|||
|
|||
try: |
|||
with open(file_name_save, encoding="gbk") as f: |
|||
content = f.read() |
|||
except: |
|||
with open(file_name_save, encoding="utf-8") as f: |
|||
content = f.read() |
|||
# elif file_name.split(".")[-1] == "docx": |
|||
# content = docx2txt.process(file_name_save) |
|||
|
|||
content_list = [i for i in content.split("\n")] |
|||
|
|||
return content_list |
|||
|
|||
|
|||
def main(question, db_type, top): |
|||
db_dict = { |
|||
"1": "yetianshi" |
|||
} |
|||
''' |
|||
定义文件路径 |
|||
''' |
|||
|
|||
''' |
|||
加载文件 |
|||
''' |
|||
|
|||
''' |
|||
文本分割 |
|||
''' |
|||
|
|||
''' |
|||
构建向量数据库 |
|||
1. 正常匹配 |
|||
2. 把文本使用大模型生成一个问题之后再进行匹配 |
|||
''' |
|||
|
|||
''' |
|||
根据提问匹配上下文 |
|||
''' |
|||
d = 1024 |
|||
db_type_list = db_type.split(",") |
|||
|
|||
paper_list_str = "" |
|||
for i in db_type_list: |
|||
embs = shengcehng_array([question]) |
|||
index = faiss.IndexFlatL2(d) # buid the index |
|||
data_np = np.load(f"data_np/{i}.npy") |
|||
data_str = open(f"data_file/{i}.txt").read().split("\n") |
|||
index.add(data_np) |
|||
D, I = index.search(embs, int(top)) |
|||
print(I) |
|||
|
|||
reference_list = [] |
|||
for i in I[0]: |
|||
reference_list.append(data_str[i]) |
|||
|
|||
for i,j in enumerate(reference_list): |
|||
paper_list_str += "第{}篇\n{}\n".format(str(i+1), j) |
|||
|
|||
''' |
|||
构造prompt |
|||
''' |
|||
print("paper_list_str", paper_list_str) |
|||
propmt_connect_ziliao_input = [] |
|||
for i in db_type_list: |
|||
propmt_connect_ziliao_input.append(propmt_connect_ziliao.format(i, paper_list_str)) |
|||
|
|||
propmt_connect_ziliao_input_str = "，".join(propmt_connect_ziliao_input) |
|||
propmt_connect_input = propmt_connect.format(question, propmt_connect_ziliao_input_str) |
|||
''' |
|||
生成回答 |
|||
''' |
|||
url_predict = "http://192.168.31.74:26000/predict" |
|||
url_search = "http://192.168.31.74:26000/search" |
|||
|
|||
# data = { |
|||
# "content": propmt_connect_input |
|||
# } |
|||
data = { |
|||
"content": propmt_connect_input, |
|||
"model": "qwq-32", |
|||
"top_p": 0.9, |
|||
"temperature": 0.6 |
|||
} |
|||
|
|||
res = dialog_line_parse(url_predict, data) |
|||
id_ = res["texts"]["id"] |
|||
|
|||
data = { |
|||
"id": id_ |
|||
} |
|||
|
|||
while True: |
|||
res = dialog_line_parse(url_search, data) |
|||
if res["code"] == 200: |
|||
break |
|||
else: |
|||
time.sleep(1) |
|||
spilt_str = "</think>" |
|||
think, response = str(res["text"]).split(spilt_str) |
|||
return think, response |
|||
|
|||
|
|||
@app.route("/upload_file", methods=["POST"]) |
|||
def upload_file(): |
|||
print(request.remote_addr) |
|||
file = request.files.get('file') |
|||
title = request.form.get("title") |
|||
data = ulit_request_file(file, title) |
|||
Building_vector_database("1", title, data) |
|||
return_json = { |
|||
"code": 200, |
|||
"info": "上传完成" |
|||
} |
|||
return jsonify(return_json) # 返回结果 |
|||
|
|||
|
|||
@app.route("/search", methods=["POST"]) |
|||
def search(): |
|||
print(request.remote_addr) |
|||
texts = request.json["texts"] |
|||
text_type = request.json["text_type"] |
|||
top = request.json["top"] |
|||
think, response = main(texts, text_type, top) |
|||
return_json = { |
|||
"code": 200, |
|||
"think": think, |
|||
"response": response |
|||
} |
|||
return jsonify(return_json) # 返回结果 |
|||
|
|||
|
|||
if __name__ == "__main__": |
|||
app.run(host="0.0.0.0", port=27000, threaded=True, debug=False) |
|||
@ -0,0 +1,179 @@ |
|||
import os |
|||
os.environ["CUDA_VISIBLE_DEVICES"] = "1,2,3,4" |
|||
from transformers import AutoModelForCausalLM, AutoTokenizer |
|||
import logging |
|||
from threading import Thread |
|||
import requests |
|||
import redis |
|||
import uuid |
|||
import time |
|||
import json |
|||
from flask import Flask, jsonify |
|||
from flask import request |
|||
|
|||
app = Flask(__name__) |
|||
app.config["JSON_AS_ASCII"] = False |
|||
pool = redis.ConnectionPool(host='localhost', port=63179, max_connections=50,db=17, password="zhicheng123*") |
|||
redis_ = redis.Redis(connection_pool=pool, decode_responses=True) |
|||
|
|||
# model config |
|||
model_name = "/home/majiahui/project/models-llm/QwQ-32" |
|||
model = AutoModelForCausalLM.from_pretrained( |
|||
model_name, |
|||
torch_dtype="auto", |
|||
device_map="auto" |
|||
) |
|||
tokenizer = AutoTokenizer.from_pretrained(model_name) |
|||
|
|||
db_key_query = 'query' |
|||
db_key_querying = 'querying' |
|||
db_key_result = 'result' |
|||
batch_size = 15 |
|||
|
|||
|
|||
def main(prompt): |
|||
# prompt = "电视剧《人世间》导演和演员是谁" |
|||
messages = [ |
|||
{"role": "user", "content": prompt} |
|||
] |
|||
text = tokenizer.apply_chat_template( |
|||
messages, |
|||
tokenize=False, |
|||
add_generation_prompt=True |
|||
) |
|||
|
|||
model_inputs = tokenizer([text], return_tensors="pt").to(model.device) |
|||
|
|||
generated_ids = model.generate( |
|||
**model_inputs, |
|||
max_new_tokens=32768 |
|||
) |
|||
generated_ids = [ |
|||
output_ids[len(input_ids):] for input_ids, output_ids in zip(model_inputs.input_ids, generated_ids) |
|||
] |
|||
|
|||
response = tokenizer.batch_decode(generated_ids, skip_special_tokens=True)[0] |
|||
response = str(response).split("</think>")[1] |
|||
return response |
|||
|
|||
|
|||
def classify(): # 调用模型，设置最大batch_size |
|||
while True: |
|||
if redis_.llen(db_key_query) == 0: # 若队列中没有元素就继续获取 |
|||
time.sleep(2) |
|||
continue |
|||
|
|||
query = redis_.lpop(db_key_query).decode('UTF-8') # 获取query的text |
|||
data_dict_path = json.loads(query) |
|||
path = data_dict_path['path'] |
|||
|
|||
with open(path, encoding='utf8') as f1: |
|||
# 加载文件的对象 |
|||
data_dict = json.load(f1) |
|||
|
|||
query_id = data_dict['id'] |
|||
input_text = data_dict["text"] |
|||
|
|||
output_text = main(input_text) |
|||
|
|||
return_text = { |
|||
"input": input_text, |
|||
"output": output_text |
|||
} |
|||
|
|||
load_result_path = "./new_data_logs/{}.json".format(query_id) |
|||
print("query_id: ", query_id) |
|||
print("load_result_path: ", load_result_path) |
|||
|
|||
with open(load_result_path, 'w', encoding='utf8') as f2: |
|||
# ensure_ascii=False才能输入中文，否则是Unicode字符 |
|||
# indent=2 JSON数据的缩进，美观 |
|||
json.dump(return_text, f2, ensure_ascii=False, indent=4) |
|||
redis_.set(query_id, load_result_path, 86400) |
|||
# log.log('start at', |
|||
# 'query_id:{},load_result_path:{},return_text:{}, debug_id_1:{}, debug_id_2:{}, debug_id_3:{}'.format( |
|||
# query_id, load_result_path, return_text) |
|||
|
|||
|
|||
@app.route("/predict", methods=["POST"]) |
|||
def predict(): |
|||
print(request.remote_addr) |
|||
content = request.json["content"] |
|||
|
|||
id_ = str(uuid.uuid1()) # 为query生成唯一标识 |
|||
print("uuid: ", id_) |
|||
d = {'id': id_, 'text': content} # 绑定文本和query id |
|||
|
|||
load_request_path = './request_data_logs/{}.json'.format(id_) |
|||
with open(load_request_path, 'w', encoding='utf8') as f2: |
|||
# ensure_ascii=False才能输入中文，否则是Unicode字符 |
|||
# indent=2 JSON数据的缩进，美观 |
|||
json.dump(d, f2, ensure_ascii=False, indent=4) |
|||
redis_.rpush(db_key_query, json.dumps({"id": id_, "path": load_request_path})) # 加入redis |
|||
redis_.sadd(db_key_querying, id_) |
|||
return_text = {"texts": {'id': id_, }, "probabilities": None, "status_code": 200} |
|||
print("ok") |
|||
return jsonify(return_text) # 返回结果 |
|||
|
|||
|
|||
@app.route("/search", methods=["POST"]) |
|||
def search(): |
|||
id_ = request.json['id'] # 获取用户query中的文本 例如"I love you" |
|||
result = redis_.get(id_) # 获取该query的模型结果 |
|||
if result is not None: |
|||
# redis_.delete(id_) |
|||
result_path = result.decode('UTF-8') |
|||
with open(result_path, encoding='utf8') as f1: |
|||
# 加载文件的对象 |
|||
result_dict = json.load(f1) |
|||
texts = result_dict["output"] |
|||
result_text = {'code': 200, 'text': texts, 'probabilities': None} |
|||
else: |
|||
querying_list = list(redis_.smembers("querying")) |
|||
querying_set = set() |
|||
for i in querying_list: |
|||
querying_set.add(i.decode()) |
|||
|
|||
querying_bool = False |
|||
if id_ in querying_set: |
|||
querying_bool = True |
|||
|
|||
query_list_json = redis_.lrange(db_key_query, 0, -1) |
|||
query_set_ids = set() |
|||
for i in query_list_json: |
|||
data_dict = json.loads(i) |
|||
query_id = data_dict['id'] |
|||
query_set_ids.add(query_id) |
|||
|
|||
query_bool = False |
|||
if id_ in query_set_ids: |
|||
query_bool = True |
|||
|
|||
if querying_bool == True and query_bool == True: |
|||
result_text = {'code': "201", 'text': "", 'probabilities': None} |
|||
elif querying_bool == True and query_bool == False: |
|||
result_text = {'code': "202", 'text': "", 'probabilities': None} |
|||
else: |
|||
result_text = {'code': "203", 'text': "", 'probabilities': None} |
|||
load_request_path = './request_data_logs_203/{}.json'.format(id_) |
|||
with open(load_request_path, 'w', encoding='utf8') as f2: |
|||
# ensure_ascii=False才能输入中文，否则是Unicode字符 |
|||
# indent=2 JSON数据的缩进，美观 |
|||
json.dump(result_text, f2, ensure_ascii=False, indent=4) |
|||
|
|||
return jsonify(result_text) # 返回结果 |
|||
|
|||
|
|||
t = Thread(target=classify) |
|||
t.start() |
|||
|
|||
if __name__ == "__main__": |
|||
logging.basicConfig(level=logging.DEBUG, # 控制台打印的日志级别 |
|||
filename='rewrite.log', |
|||
filemode='a', ##模式，有w和a，w就是写模式，每次都会重新写日志，覆盖之前的日志 |
|||
# a是追加模式，默认如果不写的话，就是追加模式 |
|||
format= |
|||
'%(asctime)s - %(pathname)s[line:%(lineno)d] - %(levelname)s: %(message)s' |
|||
# 日志格式 |
|||
) |
|||
app.run(host="0.0.0.0", port=26000, threaded=True, debug=False) |
|||
@ -0,0 +1,169 @@ |
|||
import os |
|||
os.environ["CUDA_VISIBLE_DEVICES"] = "1,2,3,4" |
|||
from transformers import AutoModelForCausalLM, AutoTokenizer |
|||
import logging |
|||
from threading import Thread |
|||
import requests |
|||
import redis |
|||
import uuid |
|||
import time |
|||
import json |
|||
from flask import Flask, jsonify |
|||
from flask import request |
|||
import deepspeed |
|||
|
|||
app = Flask(__name__) |
|||
app.config["JSON_AS_ASCII"] = False |
|||
pool = redis.ConnectionPool(host='localhost', port=63179, max_connections=50,db=17, password="zhicheng123*") |
|||
redis_ = redis.Redis(connection_pool=pool, decode_responses=True) |
|||
|
|||
# model config |
|||
model_name = "/home/majiahui/project/models-llm/QwQ-32" |
|||
|
|||
ds_config = { |
|||
"dtype": "fp16", |
|||
"tensor_parallel": { |
|||
"tp_size": 4 |
|||
} |
|||
} |
|||
|
|||
model = AutoModelForCausalLM.from_pretrained(model_name) |
|||
tokenizer = AutoTokenizer.from_pretrained(model_name) |
|||
model = deepspeed.init_inference(model, config=ds_config) |
|||
|
|||
db_key_query = 'query' |
|||
db_key_querying = 'querying' |
|||
db_key_result = 'result' |
|||
batch_size = 15 |
|||
|
|||
|
|||
def main(prompt): |
|||
# prompt = "电视剧《人世间》导演和演员是谁" |
|||
inputs = tokenizer(prompt, return_tensors="pt").to("cuda") |
|||
output = model.generate(**inputs) |
|||
response = tokenizer.decode(output[0]) |
|||
return response |
|||
|
|||
|
|||
def classify(): # 调用模型，设置最大batch_size |
|||
while True: |
|||
if redis_.llen(db_key_query) == 0: # 若队列中没有元素就继续获取 |
|||
time.sleep(2) |
|||
continue |
|||
|
|||
query = redis_.lpop(db_key_query).decode('UTF-8') # 获取query的text |
|||
data_dict_path = json.loads(query) |
|||
path = data_dict_path['path'] |
|||
|
|||
with open(path, encoding='utf8') as f1: |
|||
# 加载文件的对象 |
|||
data_dict = json.load(f1) |
|||
|
|||
query_id = data_dict['id'] |
|||
input_text = data_dict["text"] |
|||
|
|||
output_text = main(input_text) |
|||
|
|||
return_text = { |
|||
"input": input_text, |
|||
"output": output_text |
|||
} |
|||
|
|||
load_result_path = "./new_data_logs/{}.json".format(query_id) |
|||
print("query_id: ", query_id) |
|||
print("load_result_path: ", load_result_path) |
|||
|
|||
with open(load_result_path, 'w', encoding='utf8') as f2: |
|||
# ensure_ascii=False才能输入中文，否则是Unicode字符 |
|||
# indent=2 JSON数据的缩进，美观 |
|||
json.dump(return_text, f2, ensure_ascii=False, indent=4) |
|||
redis_.set(query_id, load_result_path, 86400) |
|||
# log.log('start at', |
|||
# 'query_id:{},load_result_path:{},return_text:{}, debug_id_1:{}, debug_id_2:{}, debug_id_3:{}'.format( |
|||
# query_id, load_result_path, return_text) |
|||
|
|||
|
|||
@app.route("/predict", methods=["POST"]) |
|||
def predict(): |
|||
print(request.remote_addr) |
|||
content = request.json["content"] |
|||
|
|||
id_ = str(uuid.uuid1()) # 为query生成唯一标识 |
|||
print("uuid: ", id_) |
|||
d = {'id': id_, 'text': content} # 绑定文本和query id |
|||
|
|||
load_request_path = './request_data_logs/{}.json'.format(id_) |
|||
with open(load_request_path, 'w', encoding='utf8') as f2: |
|||
# ensure_ascii=False才能输入中文，否则是Unicode字符 |
|||
# indent=2 JSON数据的缩进，美观 |
|||
json.dump(d, f2, ensure_ascii=False, indent=4) |
|||
redis_.rpush(db_key_query, json.dumps({"id": id_, "path": load_request_path})) # 加入redis |
|||
redis_.sadd(db_key_querying, id_) |
|||
return_text = {"texts": {'id': id_, }, "probabilities": None, "status_code": 200} |
|||
print("ok") |
|||
return jsonify(return_text) # 返回结果 |
|||
|
|||
|
|||
@app.route("/search", methods=["POST"]) |
|||
def search(): |
|||
id_ = request.json['id'] # 获取用户query中的文本 例如"I love you" |
|||
result = redis_.get(id_) # 获取该query的模型结果 |
|||
if result is not None: |
|||
# redis_.delete(id_) |
|||
result_path = result.decode('UTF-8') |
|||
with open(result_path, encoding='utf8') as f1: |
|||
# 加载文件的对象 |
|||
result_dict = json.load(f1) |
|||
texts = result_dict["output"] |
|||
result_text = {'code': 200, 'text': texts, 'probabilities': None} |
|||
else: |
|||
querying_list = list(redis_.smembers("querying")) |
|||
querying_set = set() |
|||
for i in querying_list: |
|||
querying_set.add(i.decode()) |
|||
|
|||
querying_bool = False |
|||
if id_ in querying_set: |
|||
querying_bool = True |
|||
|
|||
query_list_json = redis_.lrange(db_key_query, 0, -1) |
|||
query_set_ids = set() |
|||
for i in query_list_json: |
|||
data_dict = json.loads(i) |
|||
query_id = data_dict['id'] |
|||
query_set_ids.add(query_id) |
|||
|
|||
query_bool = False |
|||
if id_ in query_set_ids: |
|||
query_bool = True |
|||
|
|||
if querying_bool == True and query_bool == True: |
|||
result_text = {'code': "201", 'text': "", 'probabilities': None} |
|||
elif querying_bool == True and query_bool == False: |
|||
result_text = {'code': "202", 'text': "", 'probabilities': None} |
|||
else: |
|||
result_text = {'code': "203", 'text': "", 'probabilities': None} |
|||
load_request_path = './request_data_logs_203/{}.json'.format(id_) |
|||
with open(load_request_path, 'w', encoding='utf8') as f2: |
|||
# ensure_ascii=False才能输入中文，否则是Unicode字符 |
|||
# indent=2 JSON数据的缩进，美观 |
|||
json.dump(result_text, f2, ensure_ascii=False, indent=4) |
|||
|
|||
return jsonify(result_text) # 返回结果 |
|||
|
|||
|
|||
t = Thread(target=classify) |
|||
t.start() |
|||
|
|||
if __name__ == "__main__": |
|||
logging.basicConfig(level=logging.DEBUG, # 控制台打印的日志级别 |
|||
filename='rewrite.log', |
|||
filemode='a', ##模式，有w和a，w就是写模式，每次都会重新写日志，覆盖之前的日志 |
|||
# a是追加模式，默认如果不写的话，就是追加模式 |
|||
format= |
|||
'%(asctime)s - %(pathname)s[line:%(lineno)d] - %(levelname)s: %(message)s' |
|||
# 日志格式 |
|||
) |
|||
app.run(host="0.0.0.0", port=26000, threaded=True, debug=False) |
|||
|
|||
|
|||
@ -0,0 +1,303 @@ |
|||
import os |
|||
os.environ["CUDA_VISIBLE_DEVICES"] = "1,2,3,4" |
|||
import argparse |
|||
from typing import List, Tuple |
|||
from threading import Thread |
|||
from vllm import EngineArgs, LLMEngine, RequestOutput, SamplingParams |
|||
from transformers import AutoTokenizer |
|||
# from vllm.utils import FlexibleArgumentParser |
|||
from flask import Flask, jsonify |
|||
from flask import request |
|||
import redis |
|||
import time |
|||
import json |
|||
import uuid |
|||
|
|||
|
|||
# http接口服务 |
|||
# app = FastAPI() |
|||
app = Flask(__name__) |
|||
app.config["JSON_AS_ASCII"] = False |
|||
|
|||
pool = redis.ConnectionPool(host='localhost', port=63179, max_connections=50,db=17, password="zhicheng123*") |
|||
redis_ = redis.Redis(connection_pool=pool, decode_responses=True) |
|||
|
|||
db_key_query = 'query' |
|||
db_key_querying = 'querying' |
|||
db_key_queryset = 'queryset' |
|||
db_key_result = 'result' |
|||
db_key_error = 'error' |
|||
batch_size = 2 |
|||
tokenizer = AutoTokenizer.from_pretrained("/home/majiahui/project/models-llm/QwQ-32") |
|||
|
|||
|
|||
def smtp_f(name): |
|||
# 在下面的代码行中使用断点来调试脚本。 |
|||
import smtplib |
|||
from email.mime.text import MIMEText |
|||
from email.header import Header |
|||
|
|||
sender = '838878981@qq.com' # 发送邮箱 |
|||
receivers = ['838878981@qq.com'] # 接收邮箱 |
|||
auth_code = "jfqtutaiwrtdbcge" # 授权码 |
|||
|
|||
message = MIMEText('基础大模型出现错误，紧急', 'plain', 'utf-8') |
|||
message['From'] = Header("Sender<%s>" % sender) # 发送者 |
|||
message['To'] = Header("Receiver<%s>" % receivers[0]) # 接收者 |
|||
|
|||
subject = name |
|||
message['Subject'] = Header(subject, 'utf-8') |
|||
|
|||
try: |
|||
server = smtplib.SMTP_SSL('smtp.qq.com', 465) |
|||
server.login(sender, auth_code) |
|||
server.sendmail(sender, receivers, message.as_string()) |
|||
print("邮件发送成功") |
|||
server.close() |
|||
except smtplib.SMTPException: |
|||
print("Error: 无法发送邮件") |
|||
|
|||
class log: |
|||
def __init__(self): |
|||
pass |
|||
|
|||
def log(*args, **kwargs): |
|||
format = '%Y/%m/%d-%H:%M:%S' |
|||
format_h = '%Y-%m-%d' |
|||
value = time.localtime(int(time.time())) |
|||
dt = time.strftime(format, value) |
|||
dt_log_file = time.strftime(format_h, value) |
|||
log_file = 'log_file/access-%s' % dt_log_file + ".log" |
|||
if not os.path.exists(log_file): |
|||
with open(os.path.join(log_file), 'w', encoding='utf-8') as f: |
|||
print(dt, *args, file=f, **kwargs) |
|||
else: |
|||
with open(os.path.join(log_file), 'a+', encoding='utf-8') as f: |
|||
print(dt, *args, file=f, **kwargs) |
|||
|
|||
|
|||
def initialize_engine() -> LLMEngine: |
|||
"""Initialize the LLMEngine from the command line arguments.""" |
|||
# model_dir = "/home/majiahui/project/models-llm/Qwen-0_5B-Chat" |
|||
# model_dir = "/home/majiahui/project/models-llm/openbuddy-qwen2.5llamaify-7b_train_11_prompt_mistral_gpt_xiaobiaot_real_paper" |
|||
# model_dir = "/home/majiahui/project/models-llm/openbuddy-qwen2.5llamaify-7b_train_11_prompt_mistral_gpt_xiaobiaot_real_paper_2" |
|||
# model_dir = "/home/majiahui/project/models-llm/Qwen2.5-7B-Instruct-1M" |
|||
# model_dir = "/home/majiahui/project/models-llm/openbuddy-qwen2.5llamaify-7b-v23.1-200k" |
|||
model_dir = "/home/majiahui/project/models-llm/QwQ-32" |
|||
args = EngineArgs(model_dir) |
|||
args.max_num_seqs = 2 # batch最大20条样本 |
|||
# args.gpu_memory_utilization = 0.8 |
|||
args.tensor_parallel_size = 4 |
|||
args.max_model_len=8192 |
|||
# 加载模型 |
|||
return LLMEngine.from_engine_args(args) |
|||
|
|||
engine = initialize_engine() |
|||
|
|||
|
|||
def create_test_prompts(prompt_texts, query_ids, sampling_params_list) -> List[Tuple[str,str, SamplingParams]]: |
|||
"""Create a list of test prompts with their sampling parameters.""" |
|||
|
|||
return_list = [] |
|||
|
|||
for i,j,k in zip(prompt_texts, query_ids, sampling_params_list): |
|||
return_list.append((i, j, k)) |
|||
return return_list |
|||
|
|||
|
|||
def process_requests(engine: LLMEngine, |
|||
test_prompts: List[Tuple[str, str, SamplingParams]]): |
|||
"""Continuously process a list of prompts and handle the outputs.""" |
|||
|
|||
return_list = [] |
|||
while test_prompts or engine.has_unfinished_requests(): |
|||
if test_prompts: |
|||
prompt, query_id, sampling_params = test_prompts.pop(0) |
|||
engine.add_request(str(query_id), prompt, sampling_params) |
|||
|
|||
request_outputs: List[RequestOutput] = engine.step() |
|||
|
|||
for request_output in request_outputs: |
|||
if request_output.finished: |
|||
return_list.append(request_output) |
|||
return return_list |
|||
|
|||
|
|||
def main(prompt_texts, query_ids, sampling_params_list): |
|||
"""Main function that sets up and runs the prompt processing.""" |
|||
|
|||
test_prompts = create_test_prompts(prompt_texts, query_ids, sampling_params_list) |
|||
return process_requests(engine, test_prompts) |
|||
|
|||
|
|||
def classify(batch_size): # 调用模型，设置最大batch_size |
|||
while True: |
|||
texts = [] |
|||
query_ids = [] |
|||
sampling_params_list = [] |
|||
if redis_.llen(db_key_query) == 0: # 若队列中没有元素就继续获取 |
|||
time.sleep(2) |
|||
continue |
|||
|
|||
# for i in range(min(redis_.llen(db_key_query), batch_size)): |
|||
while True: |
|||
query = redis_.lpop(db_key_query) # 获取query的text |
|||
if query == None: |
|||
break |
|||
|
|||
query = query.decode('UTF-8') |
|||
data_dict_path = json.loads(query) |
|||
|
|||
path = data_dict_path['path'] |
|||
with open(path, encoding='utf8') as f1: |
|||
# 加载文件的对象 |
|||
data_dict = json.load(f1) |
|||
# query_ids.append(json.loads(query)['id']) |
|||
# texts.append(json.loads(query)['text']) # 拼接若干text 为batch |
|||
query_id = data_dict['id'] |
|||
print("query_id", query_id) |
|||
text = data_dict["text"] |
|||
model = data_dict["model"] |
|||
top_p = data_dict["top_p"] |
|||
temperature = data_dict["temperature"] |
|||
presence_penalty = 1.1 |
|||
max_tokens = 8192 |
|||
query_ids.append(query_id) |
|||
messages = [ |
|||
{"role": "user", "content": text} |
|||
] |
|||
text = tokenizer.apply_chat_template( |
|||
messages, |
|||
tokenize=False, |
|||
add_generation_prompt=True |
|||
) |
|||
texts.append(text) |
|||
# sampling_params = SamplingParams(temperature=0.3, top_p=0.5, stop="<|end|>", presence_penalty=1.1, max_tokens=8192) |
|||
sampling_params_list.append(SamplingParams( |
|||
temperature=temperature, |
|||
top_p=top_p, |
|||
stop="<|im_end|>", |
|||
presence_penalty=presence_penalty, |
|||
max_tokens=max_tokens |
|||
)) |
|||
if len(texts) == batch_size: |
|||
break |
|||
|
|||
print("texts", len(texts)) |
|||
print("query_ids", len(query_ids)) |
|||
print("sampling_params_list", len(sampling_params_list)) |
|||
outputs = main(texts, query_ids, sampling_params_list) |
|||
|
|||
print("预测完成") |
|||
generated_text_dict = {} |
|||
print("outputs", len(outputs)) |
|||
for i, output in enumerate(outputs): |
|||
index = output.request_id |
|||
print(index) |
|||
generated_text = output.outputs[0].text |
|||
generated_text_dict[index] = generated_text |
|||
|
|||
print(generated_text_dict) |
|||
for id_, output in generated_text_dict.items(): |
|||
|
|||
return_text = {"texts": output, "probabilities": None, "status_code": 200} |
|||
load_result_path = "./new_data_logs/{}.json".format(id_) |
|||
with open(load_result_path, 'w', encoding='utf8') as f2: |
|||
# ensure_ascii=False才能输入中文，否则是Unicode字符 |
|||
# indent=2 JSON数据的缩进，美观 |
|||
json.dump(return_text, f2, ensure_ascii=False, indent=4) |
|||
redis_.set(id_, load_result_path, 86400) |
|||
# redis_.set(id_, load_result_path, 30) |
|||
redis_.srem(db_key_querying, id_) |
|||
log.log('start at', |
|||
'query_id:{},load_result_path:{},return_text:{}'.format( |
|||
id_, load_result_path, return_text)) |
|||
|
|||
|
|||
@app.route("/predict", methods=["POST"]) |
|||
def predict(): |
|||
content = request.json["content"] # 获取用户query中的文本 例如"I love you" |
|||
model = request.json["model"] |
|||
top_p = request.json["top_p"] |
|||
temperature = request.json["temperature"] |
|||
id_ = str(uuid.uuid1()) # 为query生成唯一标识 |
|||
print("uuid: ", uuid) |
|||
d = {'id': id_, 'text': content, 'model': model, 'top_p': top_p,'temperature': temperature} # 绑定文本和query id |
|||
print(d) |
|||
try: |
|||
load_request_path = './request_data_logs/{}.json'.format(id_) |
|||
with open(load_request_path, 'w', encoding='utf8') as f2: |
|||
# ensure_ascii=False才能输入中文，否则是Unicode字符 |
|||
# indent=2 JSON数据的缩进，美观 |
|||
json.dump(d, f2, ensure_ascii=False, indent=4) |
|||
redis_.rpush(db_key_query, json.dumps({"id": id_, "path": load_request_path})) # 加入redis |
|||
redis_.sadd(db_key_querying, id_) |
|||
redis_.sadd(db_key_queryset, id_) |
|||
return_text = {"texts": {'id': id_, }, "probabilities": None, "status_code": 200} |
|||
except: |
|||
return_text = {"texts": {'id': id_, }, "probabilities": None, "status_code": 400} |
|||
smtp_f("vllm-main-paper") |
|||
return jsonify(return_text) # 返回结果 |
|||
|
|||
|
|||
@app.route("/search", methods=["POST"]) |
|||
def search(): |
|||
id_ = request.json['id'] # 获取用户query中的文本 例如"I love you" |
|||
result = redis_.get(id_) # 获取该query的模型结果 |
|||
try: |
|||
if result is not None: |
|||
result_path = result.decode('UTF-8') |
|||
with open(result_path, encoding='utf8') as f1: |
|||
# 加载文件的对象 |
|||
result_dict = json.load(f1) |
|||
code = result_dict["status_code"] |
|||
texts = result_dict["texts"] |
|||
probabilities = result_dict["probabilities"] |
|||
if str(code) == 400: |
|||
redis_.rpush(db_key_error, json.dumps({"id": id_})) |
|||
return False |
|||
result_text = {'code': code, 'text': texts, 'probabilities': probabilities} |
|||
else: |
|||
querying_list = list(redis_.smembers(db_key_querying)) |
|||
querying_set = set() |
|||
for i in querying_list: |
|||
querying_set.add(i.decode()) |
|||
|
|||
querying_bool = False |
|||
if id_ in querying_set: |
|||
querying_bool = True |
|||
|
|||
query_list_json = redis_.lrange(db_key_query, 0, -1) |
|||
query_set_ids = set() |
|||
for i in query_list_json: |
|||
data_dict = json.loads(i) |
|||
query_id = data_dict['id'] |
|||
query_set_ids.add(query_id) |
|||
|
|||
query_bool = False |
|||
if id_ in query_set_ids: |
|||
query_bool = True |
|||
|
|||
if querying_bool == True and query_bool == True: |
|||
result_text = {'code': "201", 'text': "", 'probabilities': None} |
|||
elif querying_bool == True and query_bool == False: |
|||
result_text = {'code': "202", 'text': "", 'probabilities': None} |
|||
else: |
|||
result_text = {'code': "203", 'text': "", 'probabilities': None} |
|||
load_request_path = './request_data_logs_203/{}.json'.format(id_) |
|||
with open(load_request_path, 'w', encoding='utf8') as f2: |
|||
# ensure_ascii=False才能输入中文，否则是Unicode字符 |
|||
# indent=2 JSON数据的缩进，美观 |
|||
json.dump(result_text, f2, ensure_ascii=False, indent=4) |
|||
except: |
|||
smtp_f("vllm-main") |
|||
result_text = {'code': "400", 'text': "", 'probabilities': None} |
|||
return jsonify(result_text) # 返回结果 |
|||
|
|||
|
|||
t = Thread(target=classify, args=(batch_size,)) |
|||
t.start() |
|||
|
|||
|
|||
if __name__ == "__main__": |
|||
app.run(debug=False, host='0.0.0.0', port=26000) |
|||
Loading…
Reference in new issue