From 89ba9abd409eb5c9f6e62fc944dc18dcf7d60eaf Mon Sep 17 00:00:00 2001 From: Kento HASEGAWA Date: Sun, 12 Aug 2018 16:04:37 +0900 Subject: [PATCH] Make it possible to save all the answers sent from clients (#9) --- .gitignore | 2 + comm/server/main.py | 165 +++++++++--------- comm/server/static/css/pynq-manager.css | 2 +- comm/server/static/js/pynq-manager.js | 1 - .../templates/part_question_status.html | 46 ++++- 5 files changed, 129 insertions(+), 87 deletions(-) diff --git a/.gitignore b/.gitignore index 6f6995d..0ebe4bc 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,8 @@ .DS_Store *.txt +*.tmp +*.json # Object files *.o diff --git a/comm/server/main.py b/comm/server/main.py index 845e070..56d07ec 100644 --- a/comm/server/main.py +++ b/comm/server/main.py @@ -5,6 +5,7 @@ This is intended to run on the host server (Raspberry Pi). """ import argparse +import datetime import glob import json import os @@ -27,6 +28,25 @@ questions = None clients = None current_seed = 1 +def update_answer_list(qname): + + global app_args + global questions + + questions[qname]['answers'] = [] + + _folder_name = os.path.splitext(qname)[0] + _answers_path = "{}/{}".format(app_args['out'], _folder_name) + if os.path.isdir(_answers_path): + answer_log_file = glob.glob("{}/*.json".format(_answers_path)) + answer_log_file.sort() + answer_log_file.reverse() + + for v2 in answer_log_file: + with open(v2, "r") as fp: + answer_log = json.load(fp) + questions[qname]['answers'].append(answer_log) + @app.before_request def before_request(): global app_args @@ -57,12 +77,15 @@ def before_request(): questions[_name] = { "path": v, "status": "Not solved", - "answer": {}, + "answer": "", "queue": Queue(), "board_size": board_size, - "line_num": line_num + "line_num": line_num, + "answers": [], } + update_answer_list(_name) + # 既に回答されているものを読み込む answer_path = os.path.abspath(app_args["out"]) answer_list = glob.glob("{}/T03_A*.txt".format(answer_path)) @@ -91,15 +114,55 @@ def before_request(): def post(): global questions + global app_args _client = request.form["client"] _qname = request.form["qname"] _answer = request.form["answer"] _cputime = request.form["cputime"] - dat = {"client": _client, "answer": _answer, "cputime": _cputime} + receive_time = datetime.datetime.now() + receive_time_str = receive_time.strftime("%Y%m%d%H%M%S") + + dat = { + "client": _client, + "answer": _answer, + "cputime": _cputime, + "timestamp": receive_time.strftime("%Y/%m/%d %H:%M:%S") + } + + # Create a temporary file to execute "nlcheck.py" + tmp_path = "{}/{}".format(app_args['out'], "tmp") + if not os.path.isdir(tmp_path): + os.makedirs(tmp_path) + tmp_file_name = "{}-{}.tmp".format(receive_time_str, _client.replace(":", ".")) + tmp_file_path = "{}/{}".format(tmp_path, tmp_file_name) + with open(tmp_file_path, "w") as fp: + fp.write(_answer) + probpath = "{}/{}".format(app_args["question"], _qname) + + # Format check + cmd = "/usr/bin/python /home/pi/adc2018/conmgr/adc2018/server/nlcheck.py --input {} --target {}".format(probpath, tmp_file_path) + print("`{}`".format(cmd)) + p = subprocess.run(cmd.strip().split(" "), stdout=subprocess.PIPE) + nlcheck = p.stdout.decode().strip() + + pattern = r"judge += +\[(.+?), ([0-9.]+)\]" + m = re.match(pattern, nlcheck) + print(nlcheck, m) + if m and (m.group(1) == "True"): + dat["nlcheck"] = float(m.group(2)) + else: + dat["nlcheck"] = -1 + + save_path = "{}/{}".format(app_args['out'], os.path.splitext(_qname)[0]) + if not os.path.isdir(save_path): + os.makedirs(save_path) - questions[_qname]["queue"].put(dat) + file_name = "{}-{}.json".format(receive_time_str, _client.replace(":", ".")) + save_file_path = "{}/{}".format(save_path, file_name) + with open(save_file_path, "w") as fp: + json.dump(dat, fp, indent=4) res = {"status": "OK"} @@ -132,116 +195,54 @@ def start(): qnumber = _question_name.replace(".txt", "").replace("NL_Q", "") probpath = "{}/{}".format(app_args["question"], _question_name) tmppath = "{}/T03_A{}_tmp.txt".format(app_args["out"], qnumber) - infopath = "{}/T03_A{}_info.txt".format(app_args["out"], qnumber) outpath = "{}/T03_A{}.txt".format(app_args["out"], qnumber) res = {} if line_num >= 0: - if line_num < app_args["line_num_th"]: - # LINE_NUMが閾値未満のとき，PYNQに問題を配信して問題を解かせる - res = solve_questions(_question_name, qstr) - # 一旦回答をテンポラリファイルに保存 - with open(tmppath, "w") as fp: - fp.write(res["answer"]["answer"]) - else: - # LINE_NUMが閾値以上のとき，PYNQでは解けないのでRaspberry Piに解かせる - # 文字数多くなるとコマンドラインで載りきらないからパイプで渡す - #boardstr = BoardStr.conv_boardstr(_q_lines, 'random', current_seed) - cmd = "/opt/python3.6/bin/python3.6 /home/pi/pynq-router/solver/gen_boardstr.py -t random -s {} {} | /home/pi/pynq-router/sw_huge/sim - {} {}".format(current_seed, probpath, current_seed, tmppath) - print("`{}`".format(cmd)) - time_start = time.time() - subprocess.call(cmd.strip(), shell=True) - time_done = time.time() - elapsed = time_done - time_start - res["answer"] = {} - res["answer"]["client"] = "192.168.4.1" - res["answer"]["answer"] = "Solved on Raspberry Pi!" - res["answer"]["cputime"] = elapsed - res["status"] = "Solved on Raspberry Pi" - current_seed += 1 - - # CPU time - print("CPU_time:{}".format(res["answer"]["cputime"])) - with open(infopath, "w") as f: - f.write("CPU_time:{}\n".format(res["answer"]["cputime"])) - f.write("memory:551250\n") - - # 回答をファイルに保存するとしたらここで処理する + # LINE_NUMが閾値未満のとき，PYNQに問題を配信して問題を解かせる + res = solve_questions(_question_name, qstr) + # 整形ルーティング (再配線) する #cmd = "/home/pi/pynq-router/resolver/solver --reroute --output {} {} {}".format(outpath, probpath, tmppath) cmd = "/home/pi/adc2017/pynq-router/resolver/solver --reroute --output {} {} {}".format(outpath, probpath, tmppath) print("`{}`".format(cmd)) subprocess.call(cmd.strip().split(" ")) - # たまに整形ルーティングが失敗する - # そのときは tmp を答えファイルとしてコピーする - #if not os.path.exists(outpath) and os.path.exists(tmppath): - # cmd = "/bin/cp {} {}".format(tmppath, outpath) - # subprocess.call(cmd.strip().split(" ")) - - # 回答ファイルが正しく出力されないときは，正しく解けなかったとき - if not os.path.exists(outpath): - res["status"] = "DNF" - else: - # Format check - #cmd = "/usr/bin/python /home/pi/conmgr/adc2017/server/nlcheck.py --input {} --target {}".format(probpath, outpath) - cmd = "/usr/bin/python /home/pi/adc2018/conmgr/adc2018/server/nlcheck.py --input {} --target {}".format(probpath, outpath) - print("`{}`".format(cmd)) - subprocess.call(cmd.strip().split(" ")) - # 最終結果だけを保存 questions[_question_name]["status"] = res["status"] - questions[_question_name]["answer"] = res["answer"] return json.dumps(res) def solve_questions(qname, qstr): + """ + このメソッドでは，問題データをソルバのクライアントに送りつける． + 結果は/postに送られてくるので，結果の集計はそちらで行う． + """ global clients global questions global current_seed global app_args - def worker(host, qname, qstr, qseed, q): + def worker(host, qname, qstr, qseed, request_time): _url = "http://{}/start".format(host) try: - r = requests.post(_url, data={"client": host, "qname": qname, "question": qstr, "qseed": qseed}) + r = requests.post(_url, data={"client": host, "qname": qname, "question": qstr, "qseed": qseed, "request_time": request_time}) client_res = json.loads(r.text) - q.put(client_res) except Exception as e: sys.stderr.write(str(e) + "\n") threads = [] - q = Queue() + + request_time = time.time() for c in clients: - _th = threading.Thread(name=c, target=worker, args=(c, qname, qstr, current_seed, q)) + _th = threading.Thread(name=c, target=worker, args=(c, qname, qstr, current_seed, request_time)) _th.start() threads.append(_th) current_seed += 1 - # PYNQが解き終わるまで待つ（ここでは最大10秒） - cnt = 0 - while cnt < app_args["timeout"] and questions[qname]["queue"].qsize() < 1: - time.sleep(1) - cnt += 1 - - res = {"status": "Done", "answers": [], "answer": ""} - - while not questions[qname]["queue"].empty(): - _r = questions[qname]["queue"].get() - res["answers"].append(_r) - - # res["answers"]に，回答を得られたものの結果が，返ってきた順に入る． - # 解の品質等を決めて最終的な回答を与える場合はここで処理する（今はとりあえず最初の答え） - # TODO: 答えが無い場合の処理 - if len(res["answers"]) > 0: - res["answer"] = res["answers"][0] - else: - res["answer"] = { "client": "None", "answer": "" } - res["status"] = "DNF" - - #print(res) + res = {"status": "Processed"} return res @@ -295,11 +296,14 @@ def question_status(): global app_args global questions + global clients qname = request.args.get("qname", "") + + update_answer_list(qname) qdata = questions[qname] - return render_template("part_question_status.html", qname=qname, qdata=qdata) + return render_template("part_question_status.html", qname=qname, qdata=qdata, clients=clients) @app.route("/") def index(): @@ -321,7 +325,6 @@ if __name__ == "__main__": parser.add_argument("-c", "--client", action="store", type=str, default=None, required=True, help="Client list.") parser.add_argument("-q", "--question", action="store", type=str, default="./problems", help="Path to the question folder.") parser.add_argument("-o", "--out", action="store", type=str, default="./answers", help="Path to the output folder.") - parser.add_argument("-l", "--line-num-th", action="store", type=int, default=128, help="Line number threshold.") parser.add_argument("-t", "--timeout", action="store", type=int, default=300, help="Timeout.") parser.add_argument("--debug", action="store_true", default=False, help="Debug mode.") args = vars(parser.parse_args()) diff --git a/comm/server/static/css/pynq-manager.css b/comm/server/static/css/pynq-manager.css index a909fe9..dd2cd54 100644 --- a/comm/server/static/css/pynq-manager.css +++ b/comm/server/static/css/pynq-manager.css @@ -43,7 +43,7 @@ body{ #client-control-pane{ height: 330px; - overflow: scroll; + overflow-y: scroll; } #client-control-pane table th, diff --git a/comm/server/static/js/pynq-manager.js b/comm/server/static/js/pynq-manager.js index 5bd3a91..82deab0 100644 --- a/comm/server/static/js/pynq-manager.js +++ b/comm/server/static/js/pynq-manager.js @@ -63,7 +63,6 @@ var PynqManager = (function(){ }).done(function(res){ var answer = res; $("#solving-question-status").text(answer["status"]); - $("#solved-result").text(answer["answer"]["answer"]); $("#solved-client").text(answer["answer"]["client"]); if (after !== null){ diff --git a/comm/server/templates/part_question_status.html b/comm/server/templates/part_question_status.html index ad90ab6..f669e44 100644 --- a/comm/server/templates/part_question_status.html +++ b/comm/server/templates/part_question_status.html @@ -1,7 +1,45 @@ -

問題 【{{qname}}】

-

- -

+
+
+

問題 【{{qname}}】

+

+ +

+
+
+

処理結果

+ + + + + + {% for c in clients %} + + + + + {% endfor %} +
クライアントStatus
{{c}}
+
+
+ +
+

処理結果一覧

+ + + + + + + {% for v in qdata.answers %} + + + + + + {% endfor %} +
TimestampClientScore
{{v.timestamp}}{{v.client}}{{v.nlcheck}}
+
+

状況：{{qdata.status}}

結果

クライアント：{{qdata.answer.client}}

-- 2.22.0