jiahui
/
drop_weight_rewrite
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity
Browse Source

V1.0完成

master
majiahui@haimaqingfan.com 2 years ago
parent
ea7a513ae7
commit
adf43c822a
4 changed files with 56 additions and 24 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 3
      crontab_sh.sh
  2. 50
      flask_predict_no_batch_t5.py
  3. 15
      new_data_logs/21f95512-bd79-11ed-8961-4c77cb423b31.json
  4. 12
      redis_check_uuid.py

3
crontab_sh.sh
View File

@ -0,0 +1,3 @@
rm -rf /home/majiahui/drop_weight_rewrite/request_data_logs/*
rm -rf /home/majiahui/drop_weight_rewrite/old_data_logs/*
mv /home/majiahui/drop_weight_rewrite/new_data_logs/* /home/majiahui/drop_weight_rewrite/old_data_logs/*

50
flask_predict_no_batch_t5.py
View File

@ -272,26 +272,34 @@ def classify(): # 调用模型，设置最大batch_size
if redis_.llen(db_key_query) == 0: # 若队列中没有元素就继续获取
continue
query = redis_.lpop(db_key_query).decode('UTF-8') # 获取query的text
data_dict = json.loads(query)
data_dict_path = json.loads(query)
path = data_dict_path['path']
# text_type = data_dict["text_type"]
with open(path, encoding='utf8') as f1:
# 加载文件的对象
data_dict = json.load(f1)
query_id = data_dict['id']
texts = data_dict['text'] # 拼接若干text 为batch
texts = data_dict["text"]
text_type = data_dict["text_type"]
if isinstance(texts, dict):
texts_list = []
if texts is None:
return_text = {"texts": "输入了空值", "probabilities": None, "status_code": 402}
else:
assert text_type in ['focus', 'chapter']
if text_type == 'focus':
texts_list = main(texts)
if text_type == 'chapter':
texts_list = main(texts)
return_text = {"texts": texts_list, "probabilities": None, "status_code": 200}
assert text_type in ['focus', 'chapter']
if text_type == 'focus':
texts_list = main(texts)
elif text_type == 'chapter':
texts_list = main(texts)
else:
return_text = {"texts": "输入格式应该为字典", "probabilities": None, "status_code": 401}
texts_list = []
return_text = {"texts": texts_list, "probabilities": None, "status_code": 200}
redis_.srem(db_key_querying, query_id)
redis_.set(query_id, json.dumps(return_text, ensure_ascii=False), 28800)
load_result_path = "./new_data_logs/{}.json".format(query_id)
with open(load_result_path, 'w', encoding='utf8') as f2:
# ensure_ascii=False才能输入中文，否则是Unicode字符
# indent=2 JSON数据的缩进，美观
json.dump(return_text, f2, ensure_ascii=False, indent=4)
redis_.set(query_id, load_result_path, 28800)
@app.route("/predict", methods=["POST"])
@ -300,14 +308,20 @@ def handle_query():
texts = request.json["texts"]
text_type = request.json["text_type"]
if texts is None:
return_text = {"texts": "输入了空值", "probabilities": None, "status_code": False}
return_text = {"texts": "输入了空值", "probabilities": None, "status_code": 402}
return jsonify(return_text)
if isinstance(texts, dict):
id_ = str(uuid.uuid1()) # 为query生成唯一标识
d = {'id': id_, 'text': texts, "text_type": text_type} # 绑定文本和query id
redis_.rpush(db_key_query, json.dumps(d, ensure_ascii=False)) # 加入redis
load_request_path = './request_data_logs/{}.json'.format(id_)
with open(load_request_path, 'w', encoding='utf8') as f2:
# ensure_ascii=False才能输入中文，否则是Unicode字符
# indent=2 JSON数据的缩进，美观
json.dump(d, f2, ensure_ascii=False, indent=4)
redis_.rpush(db_key_query, json.dumps({"id":id_, "path": load_request_path})) # 加入redis
redis_.sadd(db_key_querying, id_)
return_text = d
return_text = {"texts": {'id': id_,}, "probabilities": None, "status_code": 200}
print("ok")
else:
return_text = {"texts": "输入格式应该为字典", "probabilities": None, "status_code": 401}

15
new_data_logs/21f95512-bd79-11ed-8961-4c77cb423b31.json
View File

@ -0,0 +1,15 @@
{
"texts": {
"0": "李正旺你真是傻逼讪笑,挥手道,李正旺你真是傻逼讪笑,挥手道,李正旺你真是傻逼讪笑,挥手道,李正旺你真是傻逼讪笑,挥手道,李正旺你真是傻逼讪笑,挥手道,李正旺你真是傻逼讪笑,挥手道,李正旺你真是傻逼讪笑,挥手道,李正旺你真是傻逼讪笑,挥手道,李正旺你真是傻逼讪笑,挥手道,李正旺你真是傻逼讪笑,挥手道,李正旺你真是傻逼讪笑,挥手道,李正旺你真是傻逼讪笑,挥手道,李正旺你真是傻逼讪笑,挥手道,李正旺你真是傻逼讪笑,挥手道,李正旺你真是傻逼讪笑,挥手道,李正旺你真是傻逼讪笑,挥手道,李正旺你真是傻逼讪笑,挥手道,李正旺你真是傻逼讪笑,挥手道,李正旺你真是傻逼讪笑,挥手道,李正旺你真是傻逼讪笑,挥手道,李正旺你真是傻逼讪笑,挥手道,李正旺你真是傻逼讪笑,挥手道,李正旺你真是傻逼讪笑,挥手道,李正旺你真是傻逼讪笑,挥手道,李正旺你真是傻逼讪笑,挥手道,李正旺你真是傻逼讪笑,挥手道,李正旺你真是傻逼讪笑,挥手道,李正旺你真是傻逼讪笑,挥手道,李正旺你真是傻逼讪笑,挥手道,李正旺你真是傻逼讪笑,挥手道,李正旺你真是傻逼讪笑,挥手道,李正旺你真是傻逼讪笑,挥手道,李正旺你真是傻逼讪笑,挥手道,李正旺你真是傻逼讪笑,挥手道,李正旺你真是傻逼讪笑,挥手道,李正旺你真是傻逼讪笑,挥手道,李正旺你真是傻逼讪笑,挥手道,李正旺你真是傻逼讪笑,挥手道,李正旺你真是傻逼讪笑,挥手道,李正旺你真是傻逼讪笑,挥手道,李正旺你真是傻逼讪笑,挥手道,李正旺你真是傻逼讪笑,挥手道,李正旺你真是傻逼讪笑,挥手道,李正旺你真是傻逼讪笑,挥手道,李正旺你真是傻逼讪笑,挥手道,李正旺你真是傻逼讪笑,挥手道,李正旺你真是傻逼讪笑,挥手道,李正旺你真是傻逼讪笑,挥手道,李正旺你真是傻逼讪笑,挥手道,李正旺你真是傻逼讪笑,挥手道,李正旺你真是傻逼讪笑,挥手道,李正旺你真是傻逼讪笑,挥手道,李正旺你真是傻逼讪笑,挥手道,李正旺你真是傻逼讪笑,挥手道,",
"1": "李正旺你真是傻逼讪笑,挥手道",
"2": "李正旺你真是傻逼讪笑,挥手道",
"3": "李正旺你真是傻逼讪笑,挥手道",
"4": "李正旺你真是傻逼讪笑,挥手道",
"5": "李正旺你真是傻逼讪笑,挥手道",
"6": "李正旺你真是傻逼讪笑,挥手道",
"7": "李正旺你真是傻逼讪笑,挥手道",
"8": "李正旺你真是傻逼讪笑,挥手李正旺你真是傻逼讪笑,挥手道李正旺你真是傻逼讪笑,挥手道李正旺你真是傻逼讪笑,挥手道李正旺你真是傻逼讪笑,挥手道李正旺你真是傻逼讪笑,挥手道李正旺你真是傻逼讪笑,挥手道李正旺你真是傻逼讪笑,挥手道李正旺你真是傻逼讪笑,挥手道李正旺你真是傻逼讪笑,挥手道李正旺你真是傻逼讪笑,挥手道李正旺你真是傻逼讪笑,挥手道李正旺你真是傻逼讪笑,挥手道李正旺你真是傻逼讪笑,挥手道李正旺你真是傻逼讪笑,挥手道李正旺你真是傻逼讪笑,挥手道李正旺你真是傻逼讪笑,挥手道李正旺你真是傻逼讪笑,挥手道李正旺你真是傻逼讪笑,挥手道李正旺你真是傻逼讪笑,挥手道李正旺你真是傻逼讪笑,挥手道李正旺你真是傻逼讪笑,挥手道李正旺你真是傻逼讪笑,挥手道李正旺你真是傻逼讪笑,挥手道李正旺你真是傻逼讪笑,挥手道李正旺你真是傻逼讪笑,挥手道李正旺你真是傻逼讪笑,挥手道李正旺你真是傻逼讪笑,挥手道李正旺你真是傻逼讪笑,挥手道李正旺你真是傻逼讪笑,挥手道李正旺你真是傻逼讪笑,挥手道李正旺你真是傻逼讪笑,挥手道李正旺你真是傻逼讪笑,挥手道李正旺你真是傻逼讪笑,挥手道李正旺你真是傻逼讪笑,挥手道李正旺你真是傻逼讪笑,挥手道李正旺你真是傻逼讪笑,挥手道李正旺你真是傻逼讪笑,挥手道李正旺你真是傻逼讪笑,挥手道李正旺你真是傻逼讪笑,挥手道李正旺你真是傻逼讪笑,挥手道李正旺你真是傻逼讪笑,挥手道李正旺你真是傻逼讪笑,挥手道李正旺你真是傻逼讪笑,挥手道李正旺你真是傻逼讪笑,挥手道李正旺你真是傻逼讪笑,挥手道李正旺你真是傻逼讪笑,挥手道李正旺你真是傻逼讪笑,挥手道李正旺你真是傻逼讪笑,挥手"
},
"probabilities": null,
"status_code": 200
}

12
redis_check_uuid.py
View File

@ -39,16 +39,16 @@ def handle_query():
id_ = flask.request.json['id'] # 获取用户query中的文本 例如"I love you"
result = redis_.get(id_) # 获取该query的模型结果
if result is not None:
redis_.delete(id_)
result_dict = result.decode('UTF-8')
result_dict = json.loads(result_dict)
# redis_.delete(id_)
result_path = result.decode('UTF-8')
with open(result_path, encoding='utf8') as f1:
# 加载文件的对象
result_dict = json.load(f1)
texts = result_dict["texts"]
probabilities = result_dict["probabilities"]
status_code = result_dict["status_code"]
result_text = {'code': status_code, 'text': texts, 'probabilities': probabilities}
result_text = {'code': 200, 'text': texts, 'probabilities': probabilities}
else:
querying_list = list(redis_.smembers("querying"))
querying_set = set()
for i in querying_list:
querying_set.add(i.decode())

Write Preview
Loading…
Cancel
Save