From adf43c822a56154d9c9d2614a83a47565f12aceb Mon Sep 17 00:00:00 2001 From: "majiahui@haimaqingfan.com" Date: Wed, 8 Mar 2023 17:10:13 +0800 Subject: [PATCH] =?UTF-8?q?V1.0=E5=AE=8C=E6=88=90?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- crontab_sh.sh | 3 ++ flask_predict_no_batch_t5.py | 50 ++++++++++++++-------- .../21f95512-bd79-11ed-8961-4c77cb423b31.json | 15 +++++++ redis_check_uuid.py | 12 +++--- 4 files changed, 56 insertions(+), 24 deletions(-) create mode 100644 crontab_sh.sh create mode 100644 new_data_logs/21f95512-bd79-11ed-8961-4c77cb423b31.json diff --git a/crontab_sh.sh b/crontab_sh.sh new file mode 100644 index 0000000..7760044 --- /dev/null +++ b/crontab_sh.sh @@ -0,0 +1,3 @@ +rm -rf /home/majiahui/drop_weight_rewrite/request_data_logs/* +rm -rf /home/majiahui/drop_weight_rewrite/old_data_logs/* +mv /home/majiahui/drop_weight_rewrite/new_data_logs/* /home/majiahui/drop_weight_rewrite/old_data_logs/* \ No newline at end of file diff --git a/flask_predict_no_batch_t5.py b/flask_predict_no_batch_t5.py index def0b44..860318c 100644 --- a/flask_predict_no_batch_t5.py +++ b/flask_predict_no_batch_t5.py @@ -272,26 +272,34 @@ def classify(): # 调用模型，设置最大batch_size if redis_.llen(db_key_query) == 0: # 若队列中没有元素就继续获取 continue query = redis_.lpop(db_key_query).decode('UTF-8') # 获取query的text - data_dict = json.loads(query) + data_dict_path = json.loads(query) + path = data_dict_path['path'] + # text_type = data_dict["text_type"] + + with open(path, encoding='utf8') as f1: + # 加载文件的对象 + data_dict = json.load(f1) + query_id = data_dict['id'] - texts = data_dict['text'] # 拼接若干text 为batch + texts = data_dict["text"] text_type = data_dict["text_type"] - if isinstance(texts, dict): - texts_list = [] - if texts is None: - return_text = {"texts": "输入了空值", "probabilities": None, "status_code": 402} - else: - assert text_type in ['focus', 'chapter'] - if text_type == 'focus': - texts_list = main(texts) - if text_type == 'chapter': - texts_list = main(texts) - return_text = {"texts": texts_list, "probabilities": None, "status_code": 200} + + assert text_type in ['focus', 'chapter'] + if text_type == 'focus': + texts_list = main(texts) + elif text_type == 'chapter': + texts_list = main(texts) else: - return_text = {"texts": "输入格式应该为字典", "probabilities": None, "status_code": 401} + texts_list = [] + return_text = {"texts": texts_list, "probabilities": None, "status_code": 200} redis_.srem(db_key_querying, query_id) - redis_.set(query_id, json.dumps(return_text, ensure_ascii=False), 28800) + load_result_path = "./new_data_logs/{}.json".format(query_id) + with open(load_result_path, 'w', encoding='utf8') as f2: + # ensure_ascii=False才能输入中文，否则是Unicode字符 + # indent=2 JSON数据的缩进，美观 + json.dump(return_text, f2, ensure_ascii=False, indent=4) + redis_.set(query_id, load_result_path, 28800) @app.route("/predict", methods=["POST"]) @@ -300,14 +308,20 @@ def handle_query(): texts = request.json["texts"] text_type = request.json["text_type"] if texts is None: - return_text = {"texts": "输入了空值", "probabilities": None, "status_code": False} + return_text = {"texts": "输入了空值", "probabilities": None, "status_code": 402} return jsonify(return_text) if isinstance(texts, dict): id_ = str(uuid.uuid1()) # 为query生成唯一标识 d = {'id': id_, 'text': texts, "text_type": text_type} # 绑定文本和query id - redis_.rpush(db_key_query, json.dumps(d, ensure_ascii=False)) # 加入redis + + load_request_path = './request_data_logs/{}.json'.format(id_) + with open(load_request_path, 'w', encoding='utf8') as f2: + # ensure_ascii=False才能输入中文，否则是Unicode字符 + # indent=2 JSON数据的缩进，美观 + json.dump(d, f2, ensure_ascii=False, indent=4) + redis_.rpush(db_key_query, json.dumps({"id":id_, "path": load_request_path})) # 加入redis redis_.sadd(db_key_querying, id_) - return_text = d + return_text = {"texts": {'id': id_,}, "probabilities": None, "status_code": 200} print("ok") else: return_text = {"texts": "输入格式应该为字典", "probabilities": None, "status_code": 401} diff --git a/new_data_logs/21f95512-bd79-11ed-8961-4c77cb423b31.json b/new_data_logs/21f95512-bd79-11ed-8961-4c77cb423b31.json new file mode 100644 index 0000000..58ecffa --- /dev/null +++ b/new_data_logs/21f95512-bd79-11ed-8961-4c77cb423b31.json @@ -0,0 +1,15 @@ +{ + "texts": { + "0": "李正旺你真是傻逼讪笑,挥手道,李正旺你真是傻逼讪笑,挥手道,李正旺你真是傻逼讪笑,挥手道,李正旺你真是傻逼讪笑,挥手道,李正旺你真是傻逼讪笑,挥手道,李正旺你真是傻逼讪笑,挥手道,李正旺你真是傻逼讪笑,挥手道,李正旺你真是傻逼讪笑,挥手道,李正旺你真是傻逼讪笑,挥手道,李正旺你真是傻逼讪笑,挥手道,李正旺你真是傻逼讪笑,挥手道,李正旺你真是傻逼讪笑,挥手道,李正旺你真是傻逼讪笑,挥手道,李正旺你真是傻逼讪笑,挥手道,李正旺你真是傻逼讪笑,挥手道,李正旺你真是傻逼讪笑,挥手道,李正旺你真是傻逼讪笑,挥手道,李正旺你真是傻逼讪笑,挥手道,李正旺你真是傻逼讪笑,挥手道,李正旺你真是傻逼讪笑,挥手道,李正旺你真是傻逼讪笑,挥手道,李正旺你真是傻逼讪笑,挥手道,李正旺你真是傻逼讪笑,挥手道,李正旺你真是傻逼讪笑,挥手道,李正旺你真是傻逼讪笑,挥手道,李正旺你真是傻逼讪笑,挥手道,李正旺你真是傻逼讪笑,挥手道,李正旺你真是傻逼讪笑,挥手道,李正旺你真是傻逼讪笑,挥手道,李正旺你真是傻逼讪笑,挥手道,李正旺你真是傻逼讪笑,挥手道,李正旺你真是傻逼讪笑,挥手道,李正旺你真是傻逼讪笑,挥手道,李正旺你真是傻逼讪笑,挥手道,李正旺你真是傻逼讪笑,挥手道,李正旺你真是傻逼讪笑,挥手道,李正旺你真是傻逼讪笑,挥手道,李正旺你真是傻逼讪笑,挥手道,李正旺你真是傻逼讪笑,挥手道,李正旺你真是傻逼讪笑,挥手道,李正旺你真是傻逼讪笑,挥手道,李正旺你真是傻逼讪笑,挥手道,李正旺你真是傻逼讪笑,挥手道,李正旺你真是傻逼讪笑,挥手道,李正旺你真是傻逼讪笑,挥手道,李正旺你真是傻逼讪笑,挥手道,李正旺你真是傻逼讪笑,挥手道,李正旺你真是傻逼讪笑,挥手道,李正旺你真是傻逼讪笑,挥手道,李正旺你真是傻逼讪笑,挥手道,李正旺你真是傻逼讪笑,挥手道,李正旺你真是傻逼讪笑,挥手道,李正旺你真是傻逼讪笑,挥手道,李正旺你真是傻逼讪笑,挥手道,", + "1": "李正旺你真是傻逼讪笑,挥手道", + "2": "李正旺你真是傻逼讪笑,挥手道", + "3": "李正旺你真是傻逼讪笑,挥手道", + "4": "李正旺你真是傻逼讪笑,挥手道", + "5": "李正旺你真是傻逼讪笑,挥手道", + "6": "李正旺你真是傻逼讪笑,挥手道", + "7": "李正旺你真是傻逼讪笑,挥手道", + "8": "李正旺你真是傻逼讪笑,挥手李正旺你真是傻逼讪笑,挥手道李正旺你真是傻逼讪笑,挥手道李正旺你真是傻逼讪笑,挥手道李正旺你真是傻逼讪笑,挥手道李正旺你真是傻逼讪笑,挥手道李正旺你真是傻逼讪笑,挥手道李正旺你真是傻逼讪笑,挥手道李正旺你真是傻逼讪笑,挥手道李正旺你真是傻逼讪笑,挥手道李正旺你真是傻逼讪笑,挥手道李正旺你真是傻逼讪笑,挥手道李正旺你真是傻逼讪笑,挥手道李正旺你真是傻逼讪笑,挥手道李正旺你真是傻逼讪笑,挥手道李正旺你真是傻逼讪笑,挥手道李正旺你真是傻逼讪笑,挥手道李正旺你真是傻逼讪笑,挥手道李正旺你真是傻逼讪笑,挥手道李正旺你真是傻逼讪笑,挥手道李正旺你真是傻逼讪笑,挥手道李正旺你真是傻逼讪笑,挥手道李正旺你真是傻逼讪笑,挥手道李正旺你真是傻逼讪笑,挥手道李正旺你真是傻逼讪笑,挥手道李正旺你真是傻逼讪笑,挥手道李正旺你真是傻逼讪笑,挥手道李正旺你真是傻逼讪笑,挥手道李正旺你真是傻逼讪笑,挥手道李正旺你真是傻逼讪笑,挥手道李正旺你真是傻逼讪笑,挥手道李正旺你真是傻逼讪笑,挥手道李正旺你真是傻逼讪笑,挥手道李正旺你真是傻逼讪笑,挥手道李正旺你真是傻逼讪笑,挥手道李正旺你真是傻逼讪笑,挥手道李正旺你真是傻逼讪笑,挥手道李正旺你真是傻逼讪笑,挥手道李正旺你真是傻逼讪笑,挥手道李正旺你真是傻逼讪笑,挥手道李正旺你真是傻逼讪笑,挥手道李正旺你真是傻逼讪笑,挥手道李正旺你真是傻逼讪笑,挥手道李正旺你真是傻逼讪笑,挥手道李正旺你真是傻逼讪笑,挥手道李正旺你真是傻逼讪笑,挥手道李正旺你真是傻逼讪笑,挥手道李正旺你真是傻逼讪笑,挥手道李正旺你真是傻逼讪笑,挥手" + }, + "probabilities": null, + "status_code": 200 +} \ No newline at end of file diff --git a/redis_check_uuid.py b/redis_check_uuid.py index fb0538e..411ea09 100644 --- a/redis_check_uuid.py +++ b/redis_check_uuid.py @@ -39,16 +39,16 @@ def handle_query(): id_ = flask.request.json['id'] # 获取用户query中的文本 例如"I love you" result = redis_.get(id_) # 获取该query的模型结果 if result is not None: - redis_.delete(id_) - result_dict = result.decode('UTF-8') - result_dict = json.loads(result_dict) + # redis_.delete(id_) + result_path = result.decode('UTF-8') + with open(result_path, encoding='utf8') as f1: + # 加载文件的对象 + result_dict = json.load(f1) texts = result_dict["texts"] probabilities = result_dict["probabilities"] - status_code = result_dict["status_code"] - result_text = {'code': status_code, 'text': texts, 'probabilities': probabilities} + result_text = {'code': 200, 'text': texts, 'probabilities': probabilities} else: querying_list = list(redis_.smembers("querying")) - querying_set = set() for i in querying_list: querying_set.add(i.decode())