Implement host and solver function/GUIs

adc2019system.py

@@ -33,6 +33,7 @@ def set_role(role_name, config_data):
 
 def call_api(method, cmd, params):
 
+    print(f'I: API Received: {cmd}')
     if role is not None:
         return role.call_api(method, cmd, params)
     else:

main.py

@@ -24,6 +24,14 @@ def webui_template_questions():
     else:
         return abort(404)
 
+@webui.route('/template/system-summary')
+def webui_template_workers():
+    if (adc2019system.role is not None) and (adc2019system.role.type == 'host'):
+        workers = adc2019system.role.get_workers()
+        return render_template('part_system_summary.html', workers=workers)
+    else:
+        return abort(404)
+
 @webui.route('/template/question/<name>')
 def webui_template_question_status(name=None):
 
@@ -31,8 +39,11 @@ def webui_template_question_status(name=None):
         return abort(404)
 
     if (adc2019system.role is not None) and (adc2019system.role.type == 'host'):
-        questions = adc2019system.role.get_questions()
-        return render_template('part_question_status.html', questions=questions)
+        question = adc2019system.role.get_question(name)
+        if question is None:
+            return abort(404)
+        else:
+            return render_template('part_question_status.html', question=question)
     else:
         return abort(404)
 

roles/host.py

 import glob
 import json
+import sys
+import threading
+import time
 import requests
 
 from utils import Question
@@ -10,11 +13,13 @@ class Host(object):
         self.type = 'host'
 
         self.config = config
-        self.questions = []
-        self._load_questions(config['question_path'])
+        self.questions = dict()
+        self.worker_manager = None
+        self.request = dict()
+        self.processing_request = None
 
-        self.worker = dict()
-        self.setup_workers()
+        self._load_questions(config['question_path'])
+        self._setup_workers()
     
     def __repr__(self):
         return "Host"
@@ -28,39 +33,199 @@ class Host(object):
         for v in questions_path:
 
             question = Question(v)
+            _key = question.name
 
-            self.questions.append(question)
+            self.questions[_key] = question
     
-    def setup_workers(self):
+    def _setup_workers(self):
 
+        self.worker_manager = WorkerManager(self.config['address'])
         for v in self.config['worker']:
-            worker_name = v['name']
-            self.worker[worker_name] = Worker(v)
+            self.worker_manager.add_worker(v)
+    
+    def get_workers(self):
+        return self.worker_manager.get_workers()
+
+    def distribute_question(self, question_key):
+
+        question = self.get_question(question_key)
+        if question is None:
+            return {'status': 'key error'}
+        else:
+            request = Request(self.worker_manager, question.get_dict())
+            request_id = request.get_id()
+            self.request[request_id] = request
+
+            request.broadcast()
+            return {
+                'status': 'processed',
+                'request_id': request_id,
+                'timeout': request.timeout
+                }
 
     def get_questions(self):
         return self.questions
     
+    def get_question(self, _key):
+        if _key in self.questions:
+            return self.questions[_key]
+        else:
+            return None
+    
+    def store_solution(self, solution):
+        # request_idをチェックする機能もつくておく
+        request_id = solution['request_id']
+        if request_id in self.request:
+            self.request[request_id].store_response(solution)
+        else:
+            print(f'W: Unknown request_id: {request_id}')
+
+        question_key = solution['question']
+
+        if question_key in self.questions:
+            self.questions[question_key].put_solution(solution)
+            return {'status': 'registered'}
+        else:
+            return {'status': 'error'}
+    
+    def get_request_status(self, request_id):
+        if request_id in self.request:
+            return self.request[request_id].get_status()
+        else:
+            return {'status': 'unknown request'}
+
     def call_api(self, method, cmd, params):
         if cmd == 'role':
+            # サーバの役割確認
             return {'role': self.type}
+        elif cmd == 'question/solve':
+            # params['question']に指定された問題をworkerに配信
+            question_key = params['question']
+            return self.distribute_question(question_key)
+        elif cmd == 'question/solution':
+            self.store_solution(params)
+            return {'status': 'received'}
+        elif cmd == 'request/status':
+            request_id = float(params['request_id'])
+            return self.get_request_status(request_id)
         else:
             return None
 
+class WorkerManager(object):
+
+    def __init__(self, host_address):
+        self.workers = dict()
+        self.host = host_address
+    
+    def add_worker(self, conf):
+
+        worker_conf = dict()
+        worker_conf.update(conf)
+        worker_conf['host'] = self.host
+
+        worker_address = worker_conf['address']
+        self.workers[worker_address] = Worker(worker_conf)
+    
+    def get_workers(self):
+        return self.workers
+
+    def broadcast(self, cmd, params):
+
+        threads = []
+
+        def _sender(_worker, _cmd, _params):
+            _worker.post(_cmd, _params)
+
+        for k, v in self.workers.items():
+            _th = threading.Thread(name=v.address, target=_sender, args=(v, cmd, params), daemon=True)
+            _th.start()
+            threads.append(_th)
+
 class Worker(object):
 
     def __init__(self, params):
         self.address = params['address']
         self.name = params['name']
+        self.host = params['host']
         self.role = None
+        self.params = params
+        self.status = 'Setting up'
 
         self.set_role(params['role'])
     
     def post(self, path, data):
-        response = requests.post(
-            f'http://{self.address}/api/{path}',
-            json.dumps(data),
-            headers={'Content-Type': 'application/json'})
+        try:
+            response = requests.post(
+                f'http://{self.address}/api/{path}',
+                json.dumps(data),
+                headers={'Content-Type': 'application/json'},
+                timeout=2)
+            print(f"I: Post to {self.address}, API Cmd: {path}")
+        except Exception as e:
+            # sys.stderr.write(str(e) + "\n")
+            print(f"W: Failed to connect {self.address}")
+            self.status = 'Not connected'
+            response = None
         return response
     
     def set_role(self, role):
-        r = self.post('role', {'role': role})
+        r = self.post('role', self.params)
+
+class Request(object):
+
+    def __init__(self, worker_manager, data, timeout=10):
+        self.worker_manager = worker_manager
+
+        self.data = data
+        self.timeout = timeout
+    
+        self.request_id = time.time()
+
+        self.broadcast_time = None
+
+        self.response = dict()
+
+    @property
+    def request_data(self):
+        data = self.data
+        data['request_id'] = self.request_id
+        data['timeout'] = self.timeout
+        return data
+
+    def get_id(self):
+        return self.request_id
+
+    def get_dict(self):
+        return self.request_data
+    
+    def store_response(self, data):
+        # TODO: 1つのrequest_idに対し同一のworkerから2つ以上答えが返ってきた場合の例外処理
+        worker = data['worker']
+        self.response[worker] = data
+
+    def get_status(self):
+        all_workers = self.worker_manager.get_workers().keys()
+        worker_count = 0
+        response_count = 0
+        for v in all_workers:
+            worker_count += 1
+            if v in self.response:
+                response_count += 1
+        
+        status = ''
+        if worker_count == response_count:
+            status = 'done'
+        elif time.time() - self.broadcast_time > self.timeout:
+            status = 'timeout'
+        else:
+            status = 'processing'
+
+        return {
+            'status': status,
+            'workers': worker_count,
+            'solutions': response_count
+            }
+
+    def broadcast(self):
+        self.worker_manager.broadcast('solve', self.request_data)
+        self.broadcast_time = time.time()

roles/solver.py

+import importlib
+import json
+import os
+import requests
+import sys
+import time
+import threading
+
+class StoppableThread(threading.Thread):
+
+    def __init__(self, target, args=()):
+        super(StoppableThread, self).__init__(target=target, args=args)
+        self._status = 'running'
+    def stop(self):
+        if self._status == 'running':
+            self._status = 'stopping'
+    def stopped(self):
+        self._status = 'stopped'
+    def is_running(self):
+        return (self._status == 'running')
+    def is_stopping(self):
+        return (self._status == 'stopping')
+    def is_stopped(self):
+        return (self._status == 'stopped')
+
 class Solver(object):
 
     def __init__(self, config):
         self.type = 'solver'
 
+        self.host = config['host']
+        self.address = config['address']
+        self.name = config['name']
+        self.solver = importlib.import_module(f"solvers.{config['solver']}")
+        self.thread = None
+
     def __repr__(self):
         return "Solver"
+    
+    def solve(self, params):
+
+        start_time = time.time()
+        solution = self.solver.solve(params)
+        end_time = time.time()
+        self.thread.stopped()
+
+        elapsed_time = end_time - start_time
+
+        if not 'elapsed_time' in solution:
+            solution['elapsed_time'] = elapsed_time
+
+        self.submit_solution(params, solution)
+        self.thread = None
+        return True
+    
+    def post(self, path, data):
+        response = requests.post(
+            f'http://{self.host}/api/{path}',
+            json.dumps(data),
+            headers={'Content-Type': 'application/json'})
+        print(f"I: Post to {self.host}, API Cmd: {path}")
+        return response
+
+    def submit_solution(self, params, solution):
+        data = {
+            'request_id': params['request_id'],
+            'question': params['name'],
+            'worker': self.address
+        }
+        data.update(solution)
+        self.post('question/solution', data)
+
+    def start_solver(self, params):
+
+        if self.thread is None:
+            print("I: Solver started")
+            self.thread = StoppableThread(target=self.solve, args=(params, ))
+            self.thread.start()
+            return {'status': 'started'}
+        else:
+            return {'status': 'busy'}
+        
+    def stop_solver():
+
+        if self.thread is not None:
+            if self.thread.is_running():
+                self.thread.stop()
+                self.thread.join()
+                self.thread = None
 
     def call_api(self, method, cmd, params):
         if cmd == 'role':
             return {'role': self.type}
+        elif cmd == 'solve':
+            return self.start_solver(params)
         else:
             return None

solvers/SampleSolver/main.py

+def solve(params):
+    print("This is a sample solver.")
+    return {'solution': 'aaa!'}
+
+def main(params):
+    solve(a)

static/css/adc2019.css

@@ -15,30 +15,30 @@ body{
     background-color: rgba(0, 0, 50, .5);
 }
 
-#question-table-wrapper{
+#question-list-container{
     height: calc(100vh - 50px);
     overflow-y: scroll;
     overflow-x: hidden;
 }
 
-#question-table-wrapper th.large-cell,
-#question-table-wrapper td.large-cell{
+#question-list-container th.large-cell,
+#question-list-container td.large-cell{
     width: 24%;
 }
-#question-table-wrapper th.small-cell,
-#question-table-wrapper td.small-cell{
+#question-list-container th.small-cell,
+#question-list-container td.small-cell{
     width: 14%;
 }
 
-#question-table-wrapper tr.question-row,
-#question-table-wrapper tr.question-row td{
+#question-list-container tr.question-row,
+#question-list-container tr.question-row td{
     cursor: pointer;
 }
 
-#question-table-wrapper tr.question-row.q-selected{
+#question-list-container tr.question-row.q-selected{
     background-color: rgba(200, 100, 100, .3);
 }
-#question-table-wrapper tr.question-row:hover{
+#question-list-container tr.question-row:hover{
     background-color: rgba(200, 100, 100, .15);
 }
 
@@ -60,16 +60,23 @@ body{
     display: inline-block;
 }
 
-#client-control-pane tr.answer-detail-row, 
-#client-control-pane tr.answer-detail-row td{
+#client-control-pane tr.solution-detail-row, 
+#client-control-pane tr.solution-detail-row td{
     cursor: pointer;
 }
 
-#client-control-pane tr.answer-detail-row:hover{
+#client-control-pane tr.solution-detail-row:hover{
     background-color: rgba(200, 100, 100, .15);
 }
 
-#client-control-pane tr.answer-detail-row.submit-answer{
+#client-control-pane tr.solution-detail-row.submit-answer{
     background-color: rgba(100, 200, 100, .3);
 }
 
+#content-left h3{
+    display: inline-block;
+}
+
+#content-right h4.inline-heading{
+    display: inline-block;
+}

templates/index.html

@@ -12,30 +12,6 @@
     <script src="/static/js/bootstrap.bundle.min.js"></script>
     <script src="/static/js/d3.min.js"></script>
     <script src="/static/js/adc2019.js"></script>
-
-    <style>
-.axis path {
-    display: none;
-}
-.axis line {
-    stroke-opacity: 0.3;
-    shape-rendering: crispEdges;
-}
-input[type="range"] {
-    right: 0;
-    top: 0;
-    position: absolute;
-}
-svg,
-#chart-container {
-    width: 100%;
-    height: 100vh;
-    display: block;
-}
-#content-left h3{
-    display: inline-block;
-}
-</style>
 </head>
 <body>
 
@@ -44,7 +20,7 @@ svg,
     <div id="content-wrapper" class="container-fluid">
 
         <div id="content-row" class="row">
-            <div class="col-4" id="content-left">
+            <div class="col-5" id="content-left">
                 <h3>問題一覧</h3>&nbsp;
                 <span><a href="/#" id="view-server-status-button">システム状況</a></span>
                 <div id="question-list-container">
@@ -52,7 +28,7 @@ svg,
                 </div>
             </div>
 
-            <div class="col-8" id="content-right">
+            <div class="col-7" id="content-right">
                 <div id="status-container">
 
                 </div>

templates/part_question_status.html

-<div class="row">
-    <div class="col-4">
-        <h3> 【{{qname}}】</h3>
-        {#% # if localmode %#}
-        <p>
-            <button class="btn btn-primary btn-lg start-button" type="button" data-qname="{{qname}}">Start</button>
-            <button class="btn btn-danger btn-lg stop-button" type="button" data-qname="all">Stop</button>
-        </p>
-        <p>
-            <button class="btn btn-info btn-lg save-button" type="button" data-qname="{{qname}}">Save</button>
-            <button class="btn btn-success btn-lg submit-button" type="button" data-qname="{{qname}}">Up</button>
-        </p>
-        {#% else %#}
-        <!-- [View Only] -->
-        {#% endif %#}
-    </div>
-    <div class="col-8">
+<div>
+    <!-- <div class="col-4"> -->
+    <h3> 【{{question.name}}】</h3>
+    {#% # if localmode %#}
+    <p>
+        <button class="btn btn-primary btn-lg start-button" type="button" data-qname="{{question.name}}">Start</button>
+        <button class="btn btn-danger btn-lg stop-button" type="button" data-qname="all">Stop</button>
+        <button class="btn btn-info btn-lg save-button" type="button" data-qname="{{question.name}}">Save</button>
+        <button class="btn btn-success btn-lg submit-button" type="button" data-qname="{{question.name}}">Up</button>
+    </p>
+    {#% else %#}
+    <!-- [View Only] -->
+    {#% endif %#}
+    <!-- <div class="col-8">
         <p>処理結果</p>
         <table class="table table-bordered">
             <tr>
                 <th>Client (Solver)</th>
                 <th>Status</th>
             </tr>
-            <!-- {% for c in solvers %}
+            {% for c in solvers %}
             <tr>
                 <td>
                     {% if c|length > 3 %}
@@ -34,9 +31,9 @@
                 </td>
                 <td>{{qdata.solver[c[0]]}}</td>
             </tr>
-            {% endfor %} -->
+            {% endfor %}
         </table>
-    </div>
+    </div> -->
 </div>
 
 <div>
@@ -47,23 +44,42 @@
             <th>Client</th>
             <th>Score</th>
         </tr>
-        <!-- {#% for k, v in qdata.answers.items() %#}
+        {% for k, v in question.get_solutions().items() %}
         {#% if (qdata.best_json == k) and (v.answer != "") %#}
-        <tr class="answer-detail-row submit-answer" data-json="{#{k}#}" data-qname="{#{qname}#}">
+        <!-- <tr class="answer-detail-row submit-answer" data-json="{#{k}#}" data-qname="{#{qname}#}"> -->
         {#% else %#}
-        <tr class="answer-detail-row" data-json="{#{k}#}" data-qname="{#{qname}#}">
+        <tr class="solution-detail-row" data-json="{{k}}" data-qname="{{question.name}}">
         {#% endif %#}
-            <td>{#{v.timestamp}#}</td>
-            <td>{#{v.solver}#}</td>
+            <td>{{v.timestamp_str}}</td>
+            <td>{{v.worker}}</td>
             <td>
                 {#% if v.nlcheck == -1 %#}
-                Not solved
+                <!-- Not solved -->
                 {#% else %#}
                 {#{v.nlcheck}#}
                 {#% endif %#}
             </td>
         </tr>
-        {#% endfor %#} -->
+        {% endfor %}
     </table>
 </div>
 
+<div class="modal fade" id="solver-processing-modal" tabindex="-1" role="dialog" aria-labelledby="solver-processing-modal-title" aria-hidden="true">
+    <div class="modal-dialog modal-dialog-centered modal-lg" role="document">
+        <div class="modal-content">
+            <div class="modal-header">
+                <h5 class="modal-title" id="solver-processing-modal-title">{{question.name}}</h5>
+                <button type="button" class="close" data-dismiss="modal" aria-label="Close">
+                    <span aria-hidden="true">&times;</span>
+                </button>
+            </div>
+            <div class="modal-body">
+            処理中...
+            </div>
+            <div class="modal-footer">
+                <button type="button" class="btn btn-secondary" data-dismiss="modal">Close</button>
+                <button type="button" class="btn btn-primary">Save changes</button>
+            </div>
+        </div>
+    </div>
+</div>

templates/part_questions.html

@@ -2,16 +2,16 @@
     <thead>
         <tr>
             <th class="large-cell">File Name</th>
-            <th class="large-cell">Size</th>
+            <th class="small-cell">Size</th>
             <th class="small-cell">#Blocks</th>
             <th class="large-cell">Status</th>
         </tr>
     </thead>
     <tbody>
-        {% for v in questions %}
+        {% for k, v in questions.items() %}
         <tr class="question-row" data-qname="{{v.name}}">
             <td class="large-cell">{{v.name}}</td>
-            <td class="large-cell">{{v.size_str}}</td>
+            <td class="small-cell">{{v.size_str}}</td>
             <td class="small-cell">{{v.block_num}}</td>
             <td class="large-cell">{{v.status}}</td>
         </tr>

templates/part_system_summary.html

+<h3>システム状況</h3>
+
+<div class="summary-section">
+    <h4 class='inline-heading'>動作モード</h4>
+    <span>
+        {#% if local_mode %#}
+        Normal Mode
+        {#% else %#}
+        <!-- Viewer Mode -->
+        {#% endif %#}
+    </span>
+</div>
+
+<div class="summary-section">
+    {#% if local_mode %#}
+    <h4 class='inline-heading'>自動運営システム</h4>
+    <button type="button" class="btn btn-primary" id="adccli-login-button">Login</button>
+    <button type="button" class="btn btn-light" id="adccli-logout-button">Logout</button>
+    <button type="button" class="btn btn-info" id="adccli-get-all-q">問題DL</button>
+    <span id="adccli-status"></span>
+    {#% endif %#}
+</div>
+
+<div class="summary-section">
+    <h4>Worker</h4>
+    <table class="table table-bordered" id="workers-table">
+        <tr>
+            <th class="">Worker</th>
+            <th class="">Role</th>
+            <th class="">Status</th>
+        </tr>
+        {% for k, w in workers.items() %}
+        <tr class="worker-status-row" data-cname="{{w.name}}">
+            <td class="worker-status-name">
+                {{w.name}} ({{w.address}})
+            </td>
+            <td class="">{{w.role}}</td>
+            <td class="worker-status-value">{{w.status}}</td>
+        </tr>
+        {% endfor %}
+    </table>
+</div>
+

utils/data.py

+import datetime
 import os
+import time
+import uuid
 
 class Question(object):
 
@@ -9,6 +12,7 @@ class Question(object):
         self.size = (0, 0)
         self.block_num = 0
         self.status = 'Ready'
+        self.solutions = dict()
 
         self._load_question(path)
     
@@ -37,3 +41,44 @@ class Question(object):
         self.block_num = block_num
         self.name = name
         self.status = 'Ready'
+
+    def get_dict(self):
+        return {
+            'name': self.name,
+            'size': self.size,
+            'size_str': self.size_str,
+            'block_num': self.block_num,
+            'status': self.status
+        }
+
+    def put_solution(self, data):
+        solution = Solution(data)
+        solution_id = solution.get_id()
+        print(f'I: Put a solution: {solution_id}')
+        self.solutions[solution_id] = solution
+
+    def get_solutions(self):
+        return self.solutions
+
+class Solution(object):
+
+    def __init__(self, data):
+
+        self.question = data['question']
+        self.request_id = data['request_id']
+        self.worker = data['worker']
+        self.elapsed_time = data['elapsed_time']
+        self.solution = data['solution']
+
+        self.timestamp = time.time()
+        self._id = str(uuid.uuid4())
+    
+    def get_id(self):
+        return self._id
+
+    @property
+    def timestamp_str(self):
+        dt = datetime.datetime.fromtimestamp(
+            self.timestamp,
+            datetime.timezone(datetime.timedelta(hours=9)))
+        return dt.strftime('%H:%M:%S.%f')