Support for split-and-merge strategy

adc2019system.py

@@ -3,7 +3,7 @@
 
 import json
 
-from roles import Host, Solver
+from roles import Host, Solver, MergeSolver
 
 config = None
 role = None
@@ -32,7 +32,11 @@ def set_role(role_name, config_data):
     if role_name == 'host':
         role = Host(config_data)
     elif role_name == 'solver':
+        if not 'partial_mode' in config_data:
+            config['partial_mode'] = False
         role = Solver(config_data)
+    elif role_name == 'merge_solver':
+        role = MergeSolver(config_data)
 
 def call_api(method, cmd, params):
 

roles/__init__.py

 from .host import Host
 from .solver import Solver
+from .merge_solver import MergeSolver

roles/host.py

@@ -5,7 +5,7 @@ import threading
 import time
 import requests
 
-from utils import Problem
+from utils import Problem, GroupPart
 import utils.adcclilib as adccli
 
 class Host(object):
@@ -74,21 +74,45 @@ class Host(object):
             return None
     
     def store_solution(self, solution):
-        # request_idをチェックする機能もつくておく
-        request_id = solution['request_id']
-        if request_id in self.request:
-            self.request[request_id].store_response(solution)
-        else:
-            print(f'W: Unknown request_id: {request_id}')
-
-        problem_key = solution['problem']
+        
+        if 'part_id' in solution:
+            if solution['status'] != 'done':
+                return {'status': 'ignored'}
+            problem_key = solution['problem']
 
-        if problem_key in self.problems:
-            self.problems[problem_key].put_solution(solution)
+            if problem_key in self.problems:
+                res = self.problems[problem_key].put_partial_solution(solution)
+            else:
+                res = False
+            
             print(solution['solution'])
-            return {'status': 'registered'}
+            print(solution['line_map'], solution['block_map'])
+
+            if res:
+                merge_problem = self.problems[problem_key].partial_merge_problem
+                # print(merge_problem)
+                merge_problem['request_id'] = solution['request_id']
+                merge_problem['timeout'] = 10000
+                for k, v in self.worker_manager.workers.items():
+                    if v.role == 'merge_solver':
+                        v.post('solve', merge_problem)
+                return {'status': 'done'}
+
         else:
-            return {'status': 'error'}
+            request_id = solution['request_id']
+            if request_id in self.request:
+                self.request[request_id].store_response(solution)
+            else:
+                print(f'W: Unknown request_id: {request_id}')
+
+            problem_key = solution['problem']
+
+            if problem_key in self.problems:
+                self.problems[problem_key].put_solution(solution)
+                print(solution['solution'])
+                return {'status': 'registered'}
+            else:
+                return {'status': 'error'}
     
     def save_solution(self, problem_key):
 
@@ -259,7 +283,7 @@ class Worker(object):
         self.host = params['host']
         self.role = params['role']
         self.params = params
-        self.status = 'Not ready'
+        self.status = 'Ready'
 
         self.configure()
     
@@ -316,7 +340,6 @@ class Request(object):
         return self.request_data
     
     def store_response(self, data):
-        # TODO: 1つのrequest_idに対し同一のworkerから2つ以上答えが返ってきた場合の例外処理
         worker = data['worker']
         self.response[worker] = data
 

roles/merge_solver.py

+import re
+
+from roles import Solver
+
+class MergeSolver(Solver):
+
+    def __init__(self, config):
+        super().__init__(config)
+
+        self.type = 'merge_solver'
+        self.partial_mode = False
+
+    def __repr__(self):
+        return "MergeSolver"
+
+    def start_solver(self, params):
+
+        if 'merge_problem' in params:
+            return super().start_solver(params)
+        else:
+            return {'status': 'N/A'}
+
+    def submit_solution(self, params, solution):
+
+        if solution['status'] == 'done':
+            _lines = solution['solution'].splitlines()
+
+            parts_pos = list()
+
+            for _l in _lines:
+                l = _l.strip()
+                if l.startswith('SIZE'):
+                    board_size_str = l.strip().split()[1]
+                    board_solution_size = [int(v) for v in board_size_str.split('X')]
+                    bw = board_solution_size[0]
+                    bh = board_solution_size[1]
+                elif l.startswith('BLOCK'):
+                    print(l)
+                    p = r'BLOCK#([0-9]+) +@\(([0-9]+), *([0-9]+)\)'
+                    m = re.match(p, l)
+                    bi = int(m.group(1))
+                    bx = int(m.group(2))
+                    by = int(m.group(3))
+                    parts_pos.append((bx, by))
+            
+            _map = list()
+            _blocks = dict()
+            for _y in range(bh):
+                _map.append([0 for _x in range(bw)])
+
+            for (px, py), part in zip(parts_pos, params['parts']):
+                for _y, _r in enumerate(part['remap']):
+                    for _x, _c in enumerate(_r):
+                        if _c != 0:
+                            _map[py+_y][px+_x] = _c
+            
+            for (px, py), part in zip(parts_pos, params['parts']):
+                for bi, (_bx, _by) in part['remap_block'].items():
+                    _blocks[bi] = (px+_bx, py+_by)
+            
+            _sorted_blocks = sorted(_blocks.items(), key=lambda x:x[0])
+
+            solution_text = ''
+            solution_text += f'SIZE {bw}X{bh}\n'
+            for _r in _map:
+                solution_text += ','.join([str(v) for v in _r])
+                solution_text += '\n'
+            for bi, (_bx, _by) in _sorted_blocks:
+                solution_text += f'BLOCK#{bi} @({_bx},{_by})\n'
+
+            solution['solution'] = solution_text
+
+        data = {
+            'request_id': params['request_id'],
+            'problem': params['name'],
+            'worker': self.address
+        }
+        data.update(solution)
+        self.post('problem/solution', data)

roles/solver.py

@@ -5,8 +5,10 @@ import requests
 import sys
 import time
 import threading
+import traceback
 
 from collections import OrderedDict
+from utils import Problem
 
 class Solver(object):
 
@@ -16,6 +18,7 @@ class Solver(object):
         self.host = config['host']
         self.address = config['address']
         self.name = config['name']
+        self.partial_mode = config['partial_mode']
         self.solver = importlib.import_module(f"solvers.{config['solver']}")
         self.queue = OrderedDict()
 
@@ -50,6 +53,7 @@ class Solver(object):
                     self.solve(params)
                 except Exception as e:
                     print("E: An error has occurred in the solver thread")
+                    print(traceback.format_exc())
                 self.solving = None
                 self.set_status('Ready')
             else:
@@ -57,16 +61,38 @@ class Solver(object):
     
     def solve(self, params):
 
-        start_time = time.time()
-        solution = self.solver.solve(params)
-        end_time = time.time()
+        if self.partial_mode and len(params['group_problems']) > 1:
 
-        elapsed_time = end_time - start_time
+            for i, (gproblem, gline_map, gblock_map) in enumerate(params['group_problems']):
 
-        if not 'elapsed_time' in solution:
-            solution['elapsed_time'] = elapsed_time
+                data = params.copy()
+                data['problem'] = gproblem
+                
+                start_time = time.time()
+                solution = self.solver.solve(data)
+                end_time = time.time()
 
-        self.submit_solution(params, solution)
+                elapsed_time = end_time - start_time
+
+                if not 'elapsed_time' in solution:
+                    solution['elapsed_time'] = elapsed_time
+                
+                solution['part_id'] = i
+                solution['line_map'] = gline_map
+                solution['block_map'] = gblock_map
+
+                self.submit_solution(data, solution)
+        else:
+            start_time = time.time()
+            solution = self.solver.solve(params)
+            end_time = time.time()
+
+            elapsed_time = end_time - start_time
+
+            if not 'elapsed_time' in solution:
+                solution['elapsed_time'] = elapsed_time
+
+            self.submit_solution(params, solution)
 
         return True
     
@@ -114,15 +140,15 @@ class Solver(object):
             for k, v in self.queue.items():
                 if k == request_id:
                     self.queue.pop(k, None)
-            if self.solving['request_id'] == request_id:
-                self.solver.stop()()
+            if self.solving is not None and self.solving['request_id'] == request_id:
+                self.solver.stop()
         elif 'problem' in params:
             problem_name = params['problem']
             for k, v in self.queue.items():
                 if v['name'] == problem_name:
                     self.queue.pop(k, None)
-            if self.solving['name'] == problem_name:
-                self.solver.stop()()
+            if self.solving is not None and self.solving['name'] == problem_name:
+                self.solver.stop()
 
         return {'status': 'canceled'}
 

utils/__init__.py

-from .data import Problem, Solution
+from .data import Problem, Solution, GroupPart

utils/data.py

@@ -20,11 +20,13 @@ class Problem(object):
         self.tile_num = 0
         self.problem = ''
         self.solutions = dict()
+        self.partial_solutions = list()
         self.solution_path = solution_path
         self.best_solution = None
         self.line_numbers = list()
         self.connection = tuple()
         self.block_groups = list()
+        self.null_blocks = list()
 
         self._load_problem(problem_path)
 
@@ -113,7 +115,8 @@ class Problem(object):
                     block_text += ','.join(br_cells) + '\n'
                 block_text += '\n'
             
-            board_xy = math.ceil(2 * math.sqrt(num_tiles))
+            # board_xy = math.ceil(2 * math.sqrt(num_tiles))
+            board_xy = math.ceil(3 * math.sqrt(num_tiles))
             problem_text += f'SIZE {board_xy}X{board_xy}\n'
             problem_text += f'BLOCK_NUM {len(g)}\n'
             problem_text += '\n'
@@ -143,6 +146,46 @@ class Problem(object):
             _status = f'Saved'
             
         return _status
+    
+    @property
+    def partial_merge_problem(self):
+        text = ''
+        bx, by = self.size
+        bn = len(self.partial_solutions) + len(self.null_blocks)
+        # bn = len(self.partial_solutions)
+
+        text += f'SIZE {bx}X{by}\n'
+        text += f'BLOCK_NUM {bn}\n'
+        text += '\n'
+
+        parts = list()
+        part_id = 0
+
+        for v in self.partial_solutions:
+            gb = GroupPart(self, v[-1])
+            text += gb.group_block_text
+            text += '\n'
+            parts.append(gb.get_dict())
+            part_id += 1
+        
+        for v in self.null_blocks:
+            nb = NullBlockPart(self, self.blocks[v], part_id)
+            text += nb.nullblock_text
+            text += '\n'
+            parts.append(nb.get_dict())
+            part_id += 1
+        
+        return {
+            'name': self.name,
+            'size': self.size,
+            'size_str': self.size_str,
+            'block_num': bn,
+            'problem': text,
+            'group_problems': None,
+            'merge_problem': True,
+            'parts': parts,
+            'status': self.status
+        }
 
     def _load_problem(self, path):
 
@@ -190,6 +233,7 @@ class Problem(object):
                     'num_tiles': 0
                 }
                 num_block_tile = 0
+                is_null_block = True
                 for _h in range(bh):
                     li += 1
                     _l = q_lines[li].strip()
@@ -197,6 +241,7 @@ class Problem(object):
                     for v in _l.split(','):
                         _line_num = intplus(v.strip())
                         if isinstance(_line_num, int) and _line_num > 0:
+                            is_null_block = False
                             # Line number conversion
                             if not _line_num in line_number_list:
                                 line_number_list.append(_line_num)
@@ -215,6 +260,8 @@ class Problem(object):
                         _block_row.append(_line_num)
                     blocks[bi]['cells'].append(_block_row)
                 blocks[bi]['num_tiles'] = num_block_tile
+                if is_null_block:
+                    self.null_blocks.append(bi)
                 tile_num += num_block_tile
 
             li += 1
@@ -275,6 +322,7 @@ class Problem(object):
                         traverse_block_queue.put(next_block)
 
             block_groups.append(target_group_blocks)
+            self.partial_solutions.append(list())
 
         self.block_groups = block_groups
 
@@ -307,6 +355,18 @@ class Problem(object):
         # Save solution
         solution.save(self.solution_path)
     
+    def put_partial_solution(self, data):
+    
+        idx = int(data['part_id'])
+        self.partial_solutions[idx].append(data)
+        print([len(v)>0 for v in self.partial_solutions])
+        if all([len(v)>0 for v in self.partial_solutions]):
+            # for v in self.partial_solutions:
+            #     print(v[0])
+            return True
+        else:
+            return False
+    
     def save_best_solution(self):
 
         best_score = None
@@ -483,3 +543,132 @@ class Solution(object):
             self.timestamp,
             datetime.timezone(datetime.timedelta(hours=9)))
         return dt.strftime('%H:%M:%S.%f')
+
+class GroupPart(object):
+
+    def __init__(self, problem, solution):
+
+        self.problem = problem
+        self.solution = solution['solution']
+        self.part_id = solution['part_id']
+        self.line_map = solution['line_map']
+        self.block_map = solution['block_map']
+
+        self.remap = None
+        self.remap_block = dict()
+    
+    @property
+    def group_block_text(self):
+        
+        data = {
+            'problem': '',
+            'request_id': 0,
+            'worker': 'group',
+            'elapsed_time': 0,
+            'solution': self.solution,
+            'status': 'done'
+        }
+
+        s = Solution(data)
+        bx, by = s.size
+
+        text = ''
+        text += f'BLOCK#{self.part_id+1} {bx}X{by}\n'
+
+        _map = s.map
+        _remap = list()
+        for _r in s.map:
+            _remap.append([0 for v in _r])
+
+        _remap_block = dict()
+
+        # ついでにremapもやっておく
+        for _bi, _bv in s.block.items():
+            bi = self.block_map[_bi]
+            _remap_block[bi] = (_bv['x'], _bv['y'])
+            b = self.problem.blocks[bi]
+            __x = _bv['x']
+            __y = _bv['y']
+            for _cy, _cr in enumerate(b['cells']):
+                for _cx, _cc in enumerate(_cr):
+                    if _cc != 0:
+                        if _cc != '+':
+                            _remap[__y+_cy][__x+_cx] = _cc
+                        _map[__y+_cy][__x+_cx] = '+'
+        
+        for _y, _r in enumerate(_map):
+            for _x, _c in enumerate(_r):
+                l = _map[_y][_x]
+                if l != 0:
+                    if l != '+':
+                        _remap[_y][_x] = self.line_map[l]
+                    _map[_y][_x] = '+'
+            text += ','.join([str(v) for v in _map[_y]])
+            text += '\n'
+        
+        self.remap_block = _remap_block
+        self.remap = _remap
+
+        return text
+
+    def get_dict(self):
+        return {
+            'problem': self.group_block_text,
+            'part_id': self.part_id,
+            'solution': self.solution,
+            'line_map': self.line_map,
+            'block_map': self.block_map,
+            'remap_block': self.remap_block,
+            'remap': self.remap
+        }
+
+class NullBlockPart(object):
+    
+    def __init__(self, problem, nullblock, part_id):
+
+        self.problem = problem
+        self.nullblock = nullblock
+        self.part_id = part_id
+
+        self.remap = None
+        self.remap_block = dict()
+    
+    @property
+    def nullblock_text(self):
+
+        bx = self.nullblock['w']
+        by = self.nullblock['h']
+
+        text = ''
+        text += f'BLOCK#{self.part_id+1} {bx}X{by}\n'
+
+        _map = self.nullblock['cells']
+        _remap = list()
+        for _r in _map:
+            _remap.append([0 for v in _r])
+            
+        for _y, _r in enumerate(_map):
+            for _x, _c in enumerate(_r):
+                l = _map[_y][_x]
+                if l != 0:
+                    if l != '+':
+                        _remap[_y][_x] = self.line_map[l]
+                    _map[_y][_x] = '+'
+            text += ','.join([str(v) for v in _map[_y]])
+            text += '\n'
+
+        self.remap = _remap
+
+        return text
+
+    def get_dict(self):
+        bi = self.nullblock['index']
+        return {
+            'problem': self.nullblock_text,
+            'part_id': self.part_id,
+            'solution': '',
+            'line_map': [0],
+            'block_map': [0, bi],
+            'remap_block': {bi: (0, 0)},
+            'remap': self.remap
+        }