data.py

import datetime
import glob
import json
import math
import os
import queue
import re
import subprocess
import time
import uuid

class Problem(object):

    def __init__(self, problem_path, solution_path=None):

        self.path = problem_path
        self.name = ''
        self.size = (0, 0)
        self.block_num = 0
        self.blocks = dict()
        self.tile_num = 0
        self.problem = ''
        self.solutions = dict()
        self.partial_solutions = list()
        self.solution_path = solution_path
        self.best_solution = None
        self.line_numbers = list()
        self.connection = tuple()
        self.block_groups = list()
        self.null_blocks = list()

        self._load_problem(problem_path)

        if not self.solution_path is None:
            solution_files = glob.glob(f'{self.solution_path}/{self.name}/tmp-*.json')
            for v in solution_files:
                solution = Solution(solution_file=v)
                solution_id = solution.get_id()
                print(f'I: Put a solution: {solution_id}')
                self.solutions[solution_id] = solution
    
    @property
    def size_str(self):
        return f'{self.size[0]}X{self.size[1]}'
    
    @property
    def board_area(self):
        return self.size[0] * self.size[1]

    @property
    def filling_rate(self):
        if self.board_area == 0:
            return 0
        else:
            return self.tile_num / self.board_area

    @property
    def problem_text(self):
        
        def intplus(v):
            if v.isdecimal():
                return int(v)
            else:
                return v

        _text = ''

        _text += f'SIZE {self.size_str}\n'
        _text += f'BLOCK_NUM {self.block_num}\n'
        _text += '\n'

        for bi, block in self.blocks.items():
            _text += f'BLOCK#{bi} {block["w"]}X{block["h"]}\n'
            for cr in block['cells']:
                for cci, cc in enumerate(cr):
                    if cci > 0:
                        _text += ','
                    if isinstance(cc, int) and cc > 0:
                        cc = self.line_numbers.index(cc)
                    _text += f'{cc}'
                _text += '\n'
            _text += '\n'
        
        return _text

    @property
    def group_problem_text(self):

        group_problems = list()

        for g in self.block_groups:
            problem_text = ''
            num_tiles = 0
            line_remap_list = list()
            line_remap_list.append(0)
            block_remap_list = list()
            block_remap_list.append(0)
            block_text = ''
            for bi, bn in enumerate(g):
                block_remap_list.append(bn)
                b = self.blocks[bn]
                num_tiles += b['num_tiles']

                block_text += f'BLOCK#{bi+1} {b["w"]}X{b["h"]}\n'

                for br in b['cells']:
                    br_cells = list()
                    for bc in br:
                        if isinstance(bc, int) and bc > 0:
                            if not bc in line_remap_list:
                                line_remap_list.append(bc)
                            remapped_index = line_remap_list.index(bc)
                            br_cells.append(str(remapped_index))
                        else:
                            br_cells.append(str(bc))
                    block_text += ','.join(br_cells) + '\n'
                block_text += '\n'
            
            # board_xy = math.ceil(2 * math.sqrt(num_tiles))
            group_x = min(math.ceil(3 * math.sqrt(num_tiles)), self.size[0])
            group_y = min(math.ceil(3 * math.sqrt(num_tiles)), self.size[1])
            problem_text += f'SIZE {group_x}X{group_y}\n'
            problem_text += f'BLOCK_NUM {len(g)}\n'
            problem_text += '\n'
            problem_text += block_text
            
            group_problems.append((problem_text, line_remap_list, block_remap_list))

        return group_problems
    
    @property
    def status(self):
        _status = 'None'
        if self.problem != '':
            _status = 'Ready'
        if len(self.solutions) > 0:
            _solved_count = 0
            _trial_count = 0
            for k, v in self.solutions.items():
                _trial_count += 1
                if v.is_valid_solution():
                    _solved_count += 1
            if _solved_count > 0:
                _status = f'Solved ({_solved_count}/{_trial_count})'
            else:
                _status = f'Tried ({_trial_count})'
        if not self.best_solution is None:
            _status = f'Saved'
            
        return _status
    
    @property
    def partial_merge_problem(self):
        text = ''
        bx, by = self.size
        bn = len(self.partial_solutions) + len(self.null_blocks)
        # bn = len(self.partial_solutions)

        text += f'SIZE {bx}X{by}\n'
        text += f'BLOCK_NUM {bn}\n'
        text += '\n'

        parts = list()
        part_id = 0

        for v in self.partial_solutions:
            gb = GroupPart(self, v[-1])
            text += gb.group_block_text
            text += '\n'
            parts.append(gb.get_dict())
            part_id += 1
        
        for v in self.null_blocks:
            nb = NullBlockPart(self, self.blocks[v], part_id)
            text += nb.nullblock_text
            text += '\n'
            parts.append(nb.get_dict())
            part_id += 1
        
        return {
            'name': self.name,
            'size': self.size,
            'size_str': self.size_str,
            'block_num': bn,
            'problem': text,
            'group_problems': None,
            'merge_problem': True,
            'parts': parts,
            'status': self.status
        }

    def _load_problem(self, path):

        with open(path, 'r') as fp:
            q_text = fp.read()
        
        q_lines = q_text.splitlines()

        board_size = [0, 0]
        block_num = 0
        blocks = dict()

        tile_num = 0

        def intplus(v):
            if v.isdecimal():
                return int(v)
            else:
                return v

        line_number_list = [0]
        
        block_to_line = dict()
        line_to_block = dict()

        li = 0
        while li < len(q_lines):
            _l = q_lines[li]
            if "SIZE" in _l:
                board_size_str = _l.strip().split()[1]
                board_size = [int(v) for v in board_size_str.split('X')]
            if 'BLOCK_NUM' in _l:
                block_num = int(_l.strip().split()[1])
            if 'BLOCK#' in _l:
                p = r'BLOCK#([0-9]+) +([0-9]+)X([0-9]+)'
                m = re.match(p, _l.strip())
                bi = int(m.group(1))
                bw = int(m.group(2))
                bh = int(m.group(3))
                blocks[bi] = {
                    'index': bi,
                    'w': bw,
                    'h': bh,
                    'cells': list(),
                    'num_tiles': 0
                }
                num_block_tile = 0
                is_null_block = True
                for _h in range(bh):
                    li += 1
                    _l = q_lines[li].strip()
                    _block_row = []
                    for v in _l.split(','):
                        _line_num = intplus(v.strip())
                        if isinstance(_line_num, int) and _line_num > 0:
                            is_null_block = False
                            # Line number conversion
                            if not _line_num in line_number_list:
                                line_number_list.append(_line_num)
                            # _line_num = line_number_list.index(_line_num)
                            num_block_tile += 1

                            # Make connection list
                            if not bi in block_to_line:
                                block_to_line[bi] = list()
                            block_to_line[bi].append(_line_num)
                            if not _line_num in line_to_block:
                                line_to_block[_line_num] = list()
                            line_to_block[_line_num].append(bi)
                        elif _line_num == '+':
                            num_block_tile += 1
                        _block_row.append(_line_num)
                    blocks[bi]['cells'].append(_block_row)
                blocks[bi]['num_tiles'] = num_block_tile
                if is_null_block:
                    self.null_blocks.append(bi)
                tile_num += num_block_tile

            li += 1
        
        name = os.path.splitext(os.path.basename(path))[0]

        self.size = board_size
        self.block_num = block_num
        self.blocks = blocks
        self.tile_num = tile_num
        self.name = name
        self.problem = q_text
        self.line_numbers = line_number_list
        self.connection = (line_to_block, block_to_line)

        self._analyze_block_groups()

    def _analyze_block_groups(self):
                
        traversed_block = list()

        line_to_block = self.connection[0]
        block_to_line = self.connection[1]

        block_groups = list()

        for b in block_to_line.keys():

            if b in traversed_block:
                continue

            traverse_block_queue = queue.Queue()
            traverse_block_queue.put(b)

            target_group_blocks = list()

            while not traverse_block_queue.empty():

                target_block = traverse_block_queue.get()
                target_block_lines = block_to_line[target_block]

                if target_block in traversed_block:
                    continue

                traversed_block.append(target_block)
                target_group_blocks.append(target_block)

                for line in target_block_lines:
                    corresponding_blocks = line_to_block[line]
                    if corresponding_blocks[0] == target_block:
                        next_block = corresponding_blocks[1]
                    else:
                        next_block = corresponding_blocks[0]

                    if next_block in traversed_block:
                        continue
                    else:
                        traverse_block_queue.put(next_block)

            block_groups.append(target_group_blocks)
            self.partial_solutions.append(list())

        self.block_groups = block_groups

    def get_dict(self):
        return {
            'name': self.name,
            'size': self.size,
            'size_str': self.size_str,
            'block_num': self.block_num,
            'problem': self.problem,
            'group_problems': self.group_problem_text,
            'status': self.status
        }
    
    def get_d3json(self):

        return {
            'block': self.blocks,
            'w': self.size[0],
            'h': self.size[1],
            'n': self.block_num
        }

    def put_solution(self, data):
        solution = Solution(data)
        solution_id = solution.get_id()
        print(f'I: Put a solution: {solution_id}')
        self.solutions[solution_id] = solution

        if solution.status == 'done':
            outdir = f"{self.solution_path}/{self.name}"
            if not os.path.exists(outdir):
                os.mkdir(outdir)
            problem_path = self.path
            solution_path = f'{outdir}/tmp-{solution.request_id}-{solution.worker}.txt'
            with open(solution_path, 'w') as fp:
                fp.write(solution.solution)

            BASEDIR = os.path.abspath(os.path.dirname(__file__))
            NLCHECK = f"{BASEDIR}/../adc2019/server/adc2019.py"

            exec_cmd = f"python3 {NLCHECK} --Q-file {problem_path} --A-file {solution_path}".strip()
            print("ADCCLI: {}".format(exec_cmd))
            p = subprocess.run(exec_cmd, stdout=subprocess.PIPE, shell=True)
            try:
                res = float(p.stdout.decode().strip())
            except:
                res = -1
            solution.nlcheck = res
    
        # Save solution
        solution.save(self.solution_path)

    def put_partial_solution(self, data):
    
        idx = int(data['part_id'])
        self.partial_solutions[idx].append(data)
        
        outdir = f"{self.solution_path}/{self.name}"
        if not os.path.exists(outdir):
            os.mkdir(outdir)
        outpath = f"{outdir}/part-{data['request_id']}-{data['worker']}-p{data['part_id']}.json".replace(":", ".")
        with open(outpath, 'w') as fp:
            json.dump(self.get_dict(), fp, indent=4)

        if all([len(v)>0 for v in self.partial_solutions]):
            return True
        else:
            return False
    
    def save_best_solution(self):

        best_score = None
        for k, v in self.solutions.items():
            if v.is_valid_solution():
                _score = v.score
                if (best_score is None) or (_score < best_score[1]):
                    best_score = (k, _score)

        if best_score is not None:

            solution_name = self.name
            outpath = f'{self.solution_path}/submit'

            if not os.path.exists(outpath):
                os.mkdir(outpath)
            
            best_solution_key = best_score[0]

            with open(f'{outpath}/{solution_name}.txt', 'w') as fp:
                fp.write(self.solutions[best_solution_key].solution)

            with open(f'{outpath}/{solution_name}.json', 'w') as fp:
                json.dump(self.solutions[best_solution_key].get_dict(), fp, indent=4)
            
            self.best_solution = best_solution_key

    def get_solutions(self):
        return self.solutions
    
    def get_solution(self, solution_id):
        if solution_id in self.solutions:
            return self.solutions[solution_id]
        else:
            return None

class Solution(object):

    def __init__(self, data=None, solution_file=None):

        if not solution_file is None:
            with open(solution_file, 'r') as fp:
                data = json.load(fp)

        self.problem = data['problem']
        self.request_id = data['request_id']
        self.worker = data['worker']
        self.elapsed_time = data['elapsed_time']
        self.solution = data['solution']
        self.status = data['status']
        
        if 'nlcheck' in data:
            self.nlcheck = data['nlcheck']
        else:
            self.nlcheck = -1

        self.size = (None, None)
        self.map = None
        self.block = dict()

        if solution_file is None:
            self.timestamp = time.time()
            self._id = str(uuid.uuid4())
            
        else:
            self.timestamp = data['timestamp']
            self._id = data['id']

        self._parse_solution()

    def _parse_solution(self):

        if self.status != 'done':
            return

        board_size = [0, 0]
        block_num = 0

        _lines = self.solution.splitlines()

        li = 0

        bw, bh = 0, 0
        bmap = list()
        bposition = dict()

        state = 0

        while li < len(_lines):
            _l = _lines[li].strip()
            if (state == 0) and ('SIZE' in _l):
                board_size_str = _l.strip().split()[1]
                board_solution_size = [int(v) for v in board_size_str.split('X')]
                bw = board_solution_size[0]
                bh = board_solution_size[1]
                for _h in range(bh):
                    li += 1
                    _l = _lines[li].strip()
                    bmap.append([int(v.strip()) for v in _l.split(',')])
                state = 1
            if (state == 1) and ('BLOCK' in _l):
                p = r'BLOCK#([0-9]+) +@\(([0-9]+), *([0-9]+)\)'
                m = re.match(p, _l.strip())
                bi = int(m.group(1))
                bx = int(m.group(2))
                by = int(m.group(3))
                bposition[bi] = {
                    'index': bi,
                    'x': bx,
                    'y': by,
                }

            li += 1

            self.size = (bw, bh)
            self.map = bmap
            self.block = bposition

    @property
    def score(self):
        if self.is_valid_solution():
            return self.size[0] * self.size[1]
        else:
            return None
    
    @property
    def size_str(self):
        if self.is_valid_solution():
            return f'{self.size[0]}X{self.size[1]}'
        else:
            return '-'

    @property
    def nlcheck_str(self):
        return f'{self.nlcheck:.4f}'

    def is_valid_solution(self):
        return self.status == 'done'

    def get_id(self):
        return self._id
    
    def get_dict(self):
        return {
            'id': self._id,
            'timestamp': self.timestamp,
            'request_id': self.request_id,
            'worker': self.worker,
            'elapsed_time': self.elapsed_time,
            'problem': self.problem,
            'solution': self.solution,
            'status': self.status,
            'nlcheck': self.nlcheck
        }
    
    def get_d3json(self):

        if self.status == 'done':
            return {
                'w': self.size[0],
                'h': self.size[1],
                'map': self.map,
                'block': self.block
            }
        else:
            return {
                'w': 0,
                'h': 0,
                'map': [[]],
                'block': dict()
            }
    
    def save(self, basedir):
        outdir = f"{basedir}/{self.problem}"
        if not os.path.exists(outdir):
            os.mkdir(outdir)
        outpath = f"{outdir}/tmp-{self.request_id}-{self.worker}.json".replace(":", ".")
        with open(outpath, 'w') as fp:
            json.dump(self.get_dict(), fp, indent=4)

    @property
    def timestamp_str(self):
        dt = datetime.datetime.fromtimestamp(
            self.timestamp,
            datetime.timezone(datetime.timedelta(hours=9)))
        return dt.strftime('%H:%M:%S.%f')

class GroupPart(object):

    def __init__(self, problem, solution):

        self.problem = problem
        self.solution = solution['solution']
        self.part_id = solution['part_id']
        self.line_map = solution['line_map']
        self.block_map = solution['block_map']

        self.remap = None
        self.remap_block = dict()
    
    @property
    def group_block_text(self):
        
        data = {
            'problem': '',
            'request_id': 0,
            'worker': 'group',
            'elapsed_time': 0,
            'solution': self.solution,
            'status': 'done'
        }

        s = Solution(data)
        bx, by = s.size

        text = ''
        text += f'BLOCK#{self.part_id+1} {bx}X{by}\n'

        _map = s.map
        _remap = list()
        for _r in s.map:
            _remap.append([0 for v in _r])

        _remap_block = dict()

        # ついでにremapもやっておく
        for _bi, _bv in s.block.items():
            bi = self.block_map[_bi]
            _remap_block[bi] = (_bv['x'], _bv['y'])
            b = self.problem.blocks[bi]
            __x = _bv['x']
            __y = _bv['y']
            for _cy, _cr in enumerate(b['cells']):
                for _cx, _cc in enumerate(_cr):
                    if _cc != 0:
                        if _cc != '+':
                            _remap[__y+_cy][__x+_cx] = _cc
                        _map[__y+_cy][__x+_cx] = '+'
        
        for _y, _r in enumerate(_map):
            for _x, _c in enumerate(_r):
                l = _map[_y][_x]
                if l != 0:
                    if l != '+':
                        _remap[_y][_x] = self.line_map[l]
                    _map[_y][_x] = '+'
            text += ','.join([str(v) for v in _map[_y]])
            text += '\n'
        
        self.remap_block = _remap_block
        self.remap = _remap

        return text

    def get_dict(self):
        return {
            'problem': self.group_block_text,
            'part_id': self.part_id,
            'solution': self.solution,
            'line_map': self.line_map,
            'block_map': self.block_map,
            'remap_block': self.remap_block,
            'remap': self.remap
        }

class NullBlockPart(object):
    
    def __init__(self, problem, nullblock, part_id):

        self.problem = problem
        self.nullblock = nullblock
        self.part_id = part_id

        self.remap = None
        self.remap_block = dict()
    
    @property
    def nullblock_text(self):

        bx = self.nullblock['w']
        by = self.nullblock['h']

        text = ''
        text += f'BLOCK#{self.part_id+1} {bx}X{by}\n'

        _map = self.nullblock['cells']
        _remap = list()
        for _r in _map:
            _remap.append([0 for v in _r])
            
        for _y, _r in enumerate(_map):
            for _x, _c in enumerate(_r):
                l = _map[_y][_x]
                if l != 0:
                    if l != '+':
                        _remap[_y][_x] = self.line_map[l]
                    _map[_y][_x] = '+'
            text += ','.join([str(v) for v in _map[_y]])
            text += '\n'

        self.remap = _remap

        return text

    def get_dict(self):
        bi = self.nullblock['index']
        return {
            'problem': self.nullblock_text,
            'part_id': self.part_id,
            'solution': '',
            'line_map': [0],
            'block_map': [0, bi],
            'remap_block': {bi: (0, 0)},
            'remap': self.remap
        }