Merge branch 'comm-server' into 'comm'

Merge comm-server

See merge request adc2018/adc2018-system!12

comm/server/adcclilib.py

+#!/usr/bin/env python3
+
+import subprocess
+
+PYTHON2 = "/usr/bin/python"
+ADCCLI = "/home/pi/adc2018/conmgr/client/adccli"
+
+def _exec_adccli(cmd):
+    exec_cmd = "{} {} {}".format(PYTHON2, ADCCLI, cmd).strip()
+    print("ADCCLI: {}".format(exec_cmd))
+    p = subprocess.run(exec_cmd, stdout=subprocess.PIPE, shell=True)
+    res = p.stdout.decode().strip()
+    print(res)
+    return res
+
+def login(url, username, password):
+    cmd = "--URL='{}' --username='{}' --password='{}' login".format(url, username, password)
+    return _exec_adccli(cmd)
+
+def logout():
+    cmd = "logout"
+    return _exec_adccli(cmd)
+
+def whoami():
+    cmd = "whoami"
+    return _exec_adccli(cmd)
+
+def put_message(message):
+    cmd = "put-user-alive '{}'"
+    return _exec_adccli(cmd)
+
+def post_user_q(qnum, filepath):
+    cmd = "post-user-q {} {}".format(qnum, filepath)
+    return _exec_adccli(cmd)
+
+def get_q_all(outpath):
+    cmd = "get-q"
+    question_list = _exec_adccli(cmd).split("\n")
+
+    print("--- ADCCLI questions ---")
+    print(question_list)
+
+    for v in question_list:
+        if v.startswith("Q"):
+            qnumber = int(v.replace("Q", ""))
+            out_file_path = "{}/NL_Q{:02d}.txt".format(outpath, qnumber)
+            cmd = "--output {} get-q {}".format(out_file_path, qnumber)
+            r = _exec_adccli(cmd)
+
+    return question_list
+
+def put_a(qnum, filepath):
+    cmd = "put-a {} {}".format(qnum, filepath)
+    return _exec_adccli(cmd)
+
+def put_a_info(qnum, cpu, mem, misc):
+    cmd = "put-a-info {} {} {} '{}'".format(qnum, cpu, mem, misc)
+    return _exec_adccli(cmd)
+

comm/server/main.py

@@ -22,6 +22,8 @@ from queue import Queue
 sys.path.append(os.path.dirname(os.path.abspath(__file__)) + '/../../solver')
 import BoardStr
 
+import adcclilib
+
 app = Flask(__name__)
 app_args = {}
 questions = None
@@ -440,6 +442,76 @@ def get_clients():
 
     return json.dumps(res)
 
+@app.route("/adccli-login")
+def adccli_login():
+
+    global app_args
+
+    with open(app_args['adccli'], 'r') as fp:
+        d = json.load(fp)
+
+    r = adcclilib.login(d['url'], d['username'], d['password'])
+    res = {'message': r}
+
+    return json.dumps(res)
+
+@app.route("/adccli-logout")
+def adccli_logout():
+    r = adcclilib.logout()
+    res = {'message': r}
+    return json.dumps(res)
+
+@app.route("/adccli-whoami")
+def adccli_whoami():
+    global app_args
+
+    with open(app_args['adccli'], 'r') as fp:
+        d = json.load(fp)
+
+    r = adcclilib.whoami()
+    
+    res = {'message': r, 'status': r == d['username']}
+    return json.dumps(res)
+
+@app.route("/adccli-get-all-q")
+def adccli_get_all_q():
+    global app_args
+
+    out_abspath = os.path.abspath(app_args['question'])
+    r = adcclilib.get_q_all(out_abspath)
+
+    load_questions()
+
+    res = {'message': r}
+    return json.dumps(res)
+
+@app.route("/adccli-put-a", methods=["POST"])
+def adccli_put_a():
+    global app_args
+    global questions
+
+    qname = request.form["qname"]
+
+    if not "best_json" in questions[qname]:
+        res = {'message': "Required to 'save' before submission"}
+    else:
+        out_path = "{}/{}".format(app_args['out'], "submit")
+        qnumber = qname.replace("NL_Q", "").replace(".txt", "")
+        ans_file_path = "{}/T01_A{}.txt".format(out_path, qnumber)
+        ans_file_abspath = os.path.abspath(ans_file_path)
+
+        r = adcclilib.put_a(qnumber, ans_file_abspath)
+        mes = r + "\n"
+
+        json_name = questions[qname]['best_json']
+        data = questions[qname]['answers'][json_name]
+        r = adcclilib.put_a_info(qnumber, data['cputime'], "0", "Test")
+        mes += r
+
+        res = {'message': mes}
+
+    return json.dumps(res)
+
 @app.route("/board-viewer", methods=["GET"])
 def show_board_viewer():
 
@@ -524,6 +596,7 @@ if __name__ == "__main__":
 
     parser = argparse.ArgumentParser(description="PYNQ control panel.")
     parser.add_argument("-c", "--client", action="store", type=str, default=None, required=True, help="Client list.")
+    parser.add_argument("-a", "--adccli", action="store", type=str, default="./adccli.json", help="Path to the ADCCLI configuration json file.")
     parser.add_argument("-q", "--question", action="store", type=str, default="./problems", help="Path to the question folder.")
     parser.add_argument("-o", "--out", action="store", type=str, default="./answers", help="Path to the output folder.")
     parser.add_argument("--debug", action="store_true", default=False, help="Debug mode.")

comm/server/static/js/pynq-manager.js

@@ -145,6 +145,40 @@ $(function(){
         }).done(function(d){
             $("#client-control-pane").html("");
             $("#client-control-pane").html(d);
+
+            var button_action_with_ajax = function($obj, url){
+                $obj.prop("disabled", "disabled");
+                $.ajax({
+                    type: "GET",
+                    url: url,
+                    dataType: "json",
+                }).done((data)=>{
+                    $obj.prop("disabled", false);
+                    alert(data['message']);
+                });
+            };
+
+            $("#adccli-login-button").click(function(){
+                button_action_with_ajax($(this), "/adccli-login");
+            });
+            $("#adccli-logout-button").click(function(){
+                button_action_with_ajax($(this), "/adccli-logout");
+            });
+            $("#adccli-get-all-q").click(function(){
+                button_action_with_ajax($(this), "/adccli-get-all-q");
+            });
+
+            $.ajax({
+                type: "GET",
+                url: "/adccli-whoami",
+                dataType: "json",
+            }).done((data)=>{
+                if(data['status'])
+                    $("#adccli-status").text("ログイン中");
+                else
+                    $("#adccli-status").text("ログアウト");
+            });
+
             pm.getStatus();
         });
     }
@@ -177,6 +211,17 @@ $(function(){
                     alert(data['status']);
                 });
             });
+            $("#client-control-pane").find(".submit-button").eq(0).click(function(){
+                var qname = $(this).data("qname");
+                $.ajax({
+                    type: "POST",
+                    dataType: "json",
+                    url: "/adccli-put-a",
+                    data: {qname: qname}
+                }).done((data) => {
+                    alert(data['message']);
+                });
+            });
 
             $(".answer-detail-row td").click(function(){
                 var json_name = $(this).parent("tr").data("json");

comm/server/templates/part_client_table.html

@@ -9,6 +9,13 @@ Viewer Mode
 {% endif %}
 </p>
 
+<h4>自動運営システム</h4>
+<button type="button" class="btn btn-primary" id="adccli-login-button">Login</button>
+<button type="button" class="btn btn-light" id="adccli-logout-button">Logout</button>
+<span id="adccli-status"></span>
+<br />
+<button type="button" class="btn btn-info" id="adccli-get-all-q">問題DL</button>
+
 <h4>クライアント</h4>
 <table class="table table-bordered" id="clients-table">
     <tr>

comm/server/templates/part_question_status.html

@@ -3,8 +3,11 @@
         <h3> 【{{qname}}】</h3>
         <p>
             <button class="btn btn-primary btn-lg start-button" type="button" data-qname="{{qname}}">Start</button>
-            <button class="btn btn-danger btn-lg stop-button" type="button" data-qname="all">Stop</button><br />
+            <button class="btn btn-danger btn-lg stop-button" type="button" data-qname="all">Stop</button>
+        </p>
+        <p>
             <button class="btn btn-info btn-lg save-button" type="button" data-qname="{{qname}}">Save</button>
+            <button class="btn btn-success btn-lg submit-button" type="button" data-qname="{{qname}}">Up</button>
         </p>
     </div>
     <div class="col-8">

hls/ZU3EG_test/Makefile

+TARGET = sim
+OBJS = $(CPPS:.cpp=.o)
+CPPS = $(wildcard *.cpp)
+CXX = g++
+CXXFLAGS = -O3 -Wall -Wno-unknown-pragmas -Wno-unused-label -DSOFTWARE -DCALCTIME
+
+all: $(TARGET)
+
+$(TARGET): $(OBJS)
+	$(CXX) -O3 -o $@ $(OBJS)
+
+run:
+	python3 ../NLGenerator.py -x 20 -y 20 -z 6 -l 100;\
+	python3 ./gen_boardstr.py Q-20x20x5_100_10.txt |\
+	./$(TARGET) -
+
+
+clean:
+	rm *.o
+	rm $(TARGET)

hls/ZU3EG_test/README.md

+# pynq-router (sw)
+
+Vivado HLS 用 pynq-router
+
+
+## メモ
+
+* Vivado HLS 2016.1
+* Vivado 2016.1
+
+高位合成オプション:
+* Part: xc7z020clg400-1
+* Clock Period: 10.0 ns
+* Clock Uncertainty: 3.0 ns, 3.5 ns // 2017/08/04
+
+論理合成・配置配線のオプション:
+* Synthesis strategy: ~~Vivado Synthesis Defaults~~ Flow_PerfOptimized_high
+* Implementation strategy: ~~Vivado Implementation Defaults~~ Performance_Explore
+
+最適化記録：
+* 2017/08/04: クリティカルパス遅延の最適化
+  * (3.0) Estimated clock: 6.78, Max latency: 1,667,151,745 (1.00)
+  * (3.5) Estimated clock: 6.38, Max latency: 1,692,438,002 (1.00)
+* 2017/08/05: キューpop処理の動作改善
+  * (3.0) Estimated clock: 6.78, Max latency: 1,530,952,491 (0.92)
+  * (3.5) Estimated clock: 6.38, Max latency: 1,556,238,748 (0.92)
+* 2017/08/07: 剰余演算の書き換え，隣接ノードの探索ループ展開
+  * (3.0) Estimated clock: 6.78, Max latency: 1,423,592,713 (0.85)
+  * (3.5) Estimated clock: 6.38, Max latency: 1,428,235,716 (0.84)
+* 2017/08/07: 高位合成パラメータの調整 
+  * (3.0) Estimated clock: 6.78, Max latency: 1,284,364,290 (0.77)
+  * (3.5) Estimated clock: 6.38, Max latency: 1,289,003,291 (0.76)
+* 2017/08/11: 隣接ノードの探索ループ展開を戻した (そうしないと論理合成・配置配線でタイミング満たさない)
+  * (3.0) Estimated clock: 6.78, Max latency: 1,329,761,290 (0.80)
+  * (3.5) Estimated clock: 6.38, Max latency: 1,355,035,291 (0.80)
+* 2017/08/11: starts と goals を FF パーティションした
+  * (3.0) Estimated clock: 6.78, Max latency: 1,329,245,034 (0.797)
+  * (3.5) Estimated clock: 6.45, Max latency: 1,354,519,035 (0.800)
+* 2017/08/16: MAX_LINES を 256 に増やした
+  * (3.0) Estimated clock: 6.78, Max latency: 1,905,588,970 (1.143)
+  * (3.5) Estimated clock: 6.38, Max latency: 1,933,166,972 (1.142)
+* 2017/08/16: MAX_LINES を 128 に戻した (そうしないと論理合成・配置配線でタイミング満たさない)
+  * (3.0) Estimated clock: 6.78, Max latency: 1,329,245,034 (0.797)
+  * (3.5) Estimated clock: 6.45, Max latency: 1,354,519,035 (0.800)
+* 2017/08/16: コスト関数の波形をのこぎり型にした
+  * (3.0) Estimated clock: 6.78, Max latency: 1,329,245,034 (13.29 sec) (0.797)
+  * (3.25) Estimated clock: 6.52, Max latency: 1,329,249,035 (13.29 sec)
+  * (3.5) Estimated clock: 6.45, Max latency: 1,354,519,035 (13.55 sec) (0.800)
+* 2017/08/18: 対象ライン選択に剰余演算を用いない方法を試してみた ※あまり変わらないからこれはやらずに元に戻す
+  * (3.0) Estimated clock: 6.52, Max latency: 1,329,101,007 (13.29 sec) (0.797)
+  * (3.5) Estimated clock: 6.45, Max latency: 1,354,371,007 (13.54 sec) (0.800)
+* 2017/08/18: 対象ラインが連続して同じだったらルーティングスキップする
+  * (3.0) Estimated clock: 6.78, Max latency: 1,329,245,034 (13.29 sec)
+  * (3.25) Estimated clock: 6.52, Max latency: 1,329,249,035 (13.29 sec)  **←今これ**
+  * (3.5) Estimated clock: 6.45, Max latency: 1,354,519,035 (13.55 sec)
+
+
+## 入出力
+
+入力は boardstr という問題ファイルをコンパクトに記述したようなワンラインの文字列。
+
+出力は char 型の配列で、各要素に解答ファイルに相当する数値が入っている。

hls/ZU3EG_test/router.hpp

+/**
+ * router.hpp
+ *
+ * for Vivado HLS
+ */
+
+#ifndef __ROUTER_HPP__
+#define __ROUTER_HPP__
+
+#ifdef SOFTWARE
+#include "ap_int.h"
+#else
+#include <ap_int.h>
+#endif
+
+//#define DEBUG_PRINT  // いろいろ表示する
+#define USE_ASTAR    // A* 探索を使う
+#define USE_MOD_CALC // ターゲットラインの選択に剰余演算を用いる
+
+using namespace std;
+
+
+// 各種設定値
+#define MAX_WIDTH   72      // X, Y の最大値 (7ビットで収まる)
+#define BITWIDTH_XY 13
+#define BITMASK_XY  65528   // 1111 1111 1111 1000
+#define MAX_LAYER   8       // Z の最大値 (3ビット)
+#define BITWIDTH_Z  3
+#define BITMASK_Z   7       // 0000 0000 0000 0111
+
+#define MAX_CELLS  41472    // セルの総数 =72*72*8 (16ビットで収まる)
+#define MAX_LINES  1024      // ライン数の最大値 (7ビット)
+#define MAX_PATH   256      // 1つのラインが対応するセル数の最大値 (8ビット)
+#define MAX_PQ     4096     // 探索時のプライオリティ・キューの最大サイズ (12ビット) 足りないかも？
+
+#define PQ_PRIORITY_WIDTH 16
+#define PQ_PRIORITY_MASK  65535       // 0000 0000 0000 0000 1111 1111 1111 1111
+#define PQ_DATA_WIDTH     16
+#define PQ_DATA_MASK      4294901760  // 1111 1111 1111 1111 0000 0000 0000 0000
+
+#define MAX_WEIGHT 255      // 重みの最大値 (8ビットで収まる)
+#define BOARDSTR_SIZE 41472 // ボードストリングの最大文字数 (セル数 72*72*8 あれば良い)
+
+
+void lfsr_random_init(ap_uint<32> seed);
+ap_uint<32> lfsr_random();
+//ap_uint<32> lfsr_random_uint32(ap_uint<32> a, ap_uint<32> b);
+//ap_uint<32> lfsr_random_uint32_0(ap_uint<32> b);
+
+ap_uint<8> new_weight(ap_uint<16> x);
+bool pynqrouter_zu3eg_test(char boardstr[BOARDSTR_SIZE], ap_uint<32> seed, ap_int<32> *status);
+
+#ifdef USE_ASTAR
+ap_uint<7> abs_uint7(ap_uint<7> a, ap_uint<7> b);
+ap_uint<3> abs_uint3(ap_uint<3> a, ap_uint<3> b);
+#endif
+
+void search(ap_uint<8> *path_size, ap_uint<16> path[MAX_PATH], ap_uint<16> start, ap_uint<16> goal, ap_uint<8> w[MAX_WEIGHT]);
+void pq_push(ap_uint<16> priority, ap_uint<16> data, ap_uint<12> *pq_len, ap_uint<32> pq_nodes[MAX_PQ]);
+void pq_pop(ap_uint<16> *ret_priority, ap_uint<16> *ret_data, ap_uint<12> *pq_len, ap_uint<32> pq_nodes[MAX_PQ]);
+
+#endif /* __ROUTER_HPP__ */

hls/lines256_length128/Makefile

+TARGET = sim
+OBJS = $(CPPS:.cpp=.o)
+CPPS = $(wildcard *.cpp)
+CXX = g++
+CXXFLAGS = -O3 -Wall -Wno-unknown-pragmas -Wno-unused-label -DSOFTWARE -DCALCTIME
+
+all: $(TARGET)
+
+$(TARGET): $(OBJS)
+	$(CXX) -O3 -o $@ $(OBJS)
+
+run:
+	python3 ../NLGenerator.py -x 20 -y 20 -z 6 -l 100;\
+	python3 ./gen_boardstr.py Q-20x20x5_100_10.txt |\
+	./$(TARGET) -
+
+
+clean:
+	rm *.o
+	rm $(TARGET)