Initial commit

UIED/detect_compo/deprecated/Block.py

@@ -0,0 +1,56 @@
+import cv2
+from os.path import join as pjoin
+import time
+import numpy as np
+
+from detect_compo.lib_ip.Component import Component
+from config.CONFIG_UIED import Config
+C = Config()
+
+
+class Block(Component):
+    def __init__(self, region, image_shape):
+        super().__init__(region, image_shape)
+        self.category = 'Block'
+        self.parent = None
+        self.children = []
+        self.uicompo_ = None
+        self.top_or_botm = None
+        self.redundant = False
+
+    def block_is_uicompo(self, image_shape, max_compo_scale):
+        '''
+        Check the if the block is a ui component according to its relative size
+        '''
+        row, column = image_shape[:2]
+        # print(height, height / row, max_compo_scale[0], height / row > max_compo_scale[0])
+        # draw.draw_bounding_box(org, [corner], show=True)
+        # ignore atomic components
+        if self.bbox.height / row > max_compo_scale[0] or self.bbox.width / column > max_compo_scale[1]:
+            return False
+        return True
+
+    def block_is_top_or_bottom_bar(self, image_shape, top_bottom_height):
+        '''
+        Check if the block is top bar or bottom bar
+        '''
+        height, width = image_shape[:2]
+        (column_min, row_min, column_max, row_max) = self.bbox.put_bbox()
+        if column_min < 5 and row_min < 5 and \
+                width - column_max < 5 and row_max < height * top_bottom_height[0]:
+            self.uicompo_ = True
+            return True
+        if column_min < 5 and row_min > height * top_bottom_height[1] and \
+                width - column_max < 5 and height - row_max < 5:
+            self.uicompo_ = True
+            return True
+        return False
+
+    def block_erase_from_bin(self, binary, pad):
+        (column_min, row_min, column_max, row_max) = self.put_bbox()
+        column_min = max(column_min - pad, 0)
+        column_max = min(column_max + pad, binary.shape[1])
+        row_min = max(row_min - pad, 0)
+        row_max = min(row_max + pad, binary.shape[0])
+        cv2.rectangle(binary, (column_min, row_min), (column_max, row_max), (0), -1)
+

UIED/detect_compo/deprecated/block_division.py

@@ -0,0 +1,108 @@
+import cv2
+import numpy as np
+from random import randint as rint
+import time
+
+import detect_compo.lib_ip.ip_preprocessing as pre
+import detect_compo.lib_ip.ip_detection as det
+import detect_compo.lib_ip.ip_draw as draw
+import detect_compo.lib_ip.ip_segment as seg
+from detect_compo.lib_ip.Block import Block
+from config.CONFIG_UIED import Config
+C = Config()
+
+
+def block_hierarchy(blocks):
+    for i in range(len(blocks) - 1):
+        for j in range(i + 1, len(blocks)):
+            relation = blocks[i].compo_relation(blocks[j])
+            if relation == -1:
+                blocks[j].children.append(i)
+            if relation == 1:
+                blocks[i].children.append(j)
+    return
+
+
+def block_bin_erase_all_blk(binary, blocks, pad=0, show=False):
+    '''
+    erase the block parts from the binary map
+    :param binary: binary map of original image
+    :param blocks_corner: corners of detected layout block
+    :param show: show or not
+    :param pad: expand the bounding boxes of blocks
+    :return: binary map without block parts
+    '''
+
+    bin_org = binary.copy()
+    for block in blocks:
+        block.block_erase_from_bin(binary, pad)
+    if show:
+        cv2.imshow('before', bin_org)
+        cv2.imshow('after', binary)
+        cv2.waitKey()
+
+
+def block_division(grey, org, grad_thresh,
+                   show=False, write_path=None,
+                   step_h=10, step_v=10,
+                   line_thickness=C.THRESHOLD_LINE_THICKNESS,
+                   min_rec_evenness=C.THRESHOLD_REC_MIN_EVENNESS,
+                   max_dent_ratio=C.THRESHOLD_REC_MAX_DENT_RATIO,
+                   min_block_height_ratio=C.THRESHOLD_BLOCK_MIN_HEIGHT):
+    '''
+    :param grey: grey-scale of original image
+    :return: corners: list of [(top_left, bottom_right)]
+                        -> top_left: (column_min, row_min)
+                        -> bottom_right: (column_max, row_max)
+    '''
+    blocks = []
+    mask = np.zeros((grey.shape[0]+2, grey.shape[1]+2), dtype=np.uint8)
+    broad = np.zeros((grey.shape[0], grey.shape[1], 3), dtype=np.uint8)
+    broad_all = broad.copy()
+
+    row, column = grey.shape[0], grey.shape[1]
+    for x in range(0, row, step_h):
+        for y in range(0, column, step_v):
+            if mask[x, y] == 0:
+                # region = flood_fill_bfs(grey, x, y, mask)
+
+                # flood fill algorithm to get background (layout block)
+                mask_copy = mask.copy()
+                ff = cv2.floodFill(grey, mask, (y, x), None, grad_thresh, grad_thresh, cv2.FLOODFILL_MASK_ONLY)
+                # ignore small regions
+                if ff[0] < 500: continue
+                mask_copy = mask - mask_copy
+                region = np.reshape(cv2.findNonZero(mask_copy[1:-1, 1:-1]), (-1, 2))
+                region = [(p[1], p[0]) for p in region]
+
+                block = Block(region, grey.shape)
+                # draw.draw_region(region, broad_all)
+                # if block.height < 40 and block.width < 40:
+                #     continue
+                if block.height < 30:
+                    continue
+
+                # print(block.area / (row * column))
+                if block.area / (row * column) > 0.9:
+                    continue
+                elif block.area / (row * column) > 0.7:
+                    block.redundant = True
+
+                # get the boundary of this region
+                # ignore lines
+                if block.compo_is_line(line_thickness):
+                    continue
+                # ignore non-rectangle as blocks must be rectangular
+                if not block.compo_is_rectangle(min_rec_evenness, max_dent_ratio):
+                    continue
+                # if block.height/row < min_block_height_ratio:
+                #     continue
+                blocks.append(block)
+                # draw.draw_region(region, broad)
+    if show:
+        cv2.imshow('flood-fill all', broad_all)
+        cv2.imshow('block', broad)
+        cv2.waitKey()
+    if write_path is not None:
+        cv2.imwrite(write_path, broad)
+    return blocks

UIED/detect_compo/deprecated/ip_detection_utils.py

@@ -0,0 +1,461 @@
+import numpy as np
+import cv2
+from collections import Counter
+
+import lib_ip.ip_draw as draw
+from config.CONFIG_UIED import Config
+C = Config()
+
+
+# detect object(connected region)
+# def boundary_bfs_connected_area(img, x, y, mark):
+#     def neighbor(img, x, y, mark, stack):
+#         for i in range(x - 1, x + 2):
+#             if i < 0 or i >= img.shape[0]: continue
+#             for j in range(y - 1, y + 2):
+#                 if j < 0 or j >= img.shape[1]: continue
+#                 if img[i, j] == 255 and mark[i, j] == 0:
+#                     stack.append([i, j])
+#                     mark[i, j] = 255
+#
+#     stack = [[x, y]]  # points waiting for inspection
+#     area = [[x, y]]  # points of this area
+#     mark[x, y] = 255  # drawing broad
+#
+#     while len(stack) > 0:
+#         point = stack.pop()
+#         area.append(point)
+#         neighbor(img, point[0], point[1], mark, stack)
+#     return area
+
+
+# def line_check_perpendicular(lines_h, lines_v, max_thickness):
+#     """
+#     lines: [line_h, line_v]
+#         -> line_h: horizontal {'head':(column_min, row), 'end':(column_max, row), 'thickness':int)
+#         -> line_v: vertical {'head':(column, row_min), 'end':(column, row_max), 'thickness':int}
+#     """
+#     is_per_h = np.full(len(lines_h), False)
+#     is_per_v = np.full(len(lines_v), False)
+#     for i in range(len(lines_h)):
+#         # save the intersection point of h
+#         lines_h[i]['inter_point'] = set()
+#         h = lines_h[i]
+#
+#         for j in range(len(lines_v)):
+#             # save the intersection point of v
+#             if 'inter_point' not in lines_v[j]: lines_v[j]['inter_point'] = set()
+#             v = lines_v[j]
+#
+#             # if h is perpendicular to v in head of v
+#             if abs(h['head'][1]-v['head'][1]) <= max_thickness:
+#                 if abs(h['head'][0] - v['head'][0]) <= max_thickness:
+#                     lines_h[i]['inter_point'].add('head')
+#                     lines_v[j]['inter_point'].add('head')
+#                     is_per_h[i] = True
+#                     is_per_v[j] = True
+#                 elif abs(h['end'][0] - v['head'][0]) <= max_thickness:
+#                     lines_h[i]['inter_point'].add('end')
+#                     lines_v[j]['inter_point'].add('head')
+#                     is_per_h[i] = True
+#                     is_per_v[j] = True
+#
+#             # if h is perpendicular to v in end of v
+#             elif abs(h['head'][1]-v['end'][1]) <= max_thickness:
+#                 if abs(h['head'][0] - v['head'][0]) <= max_thickness:
+#                     lines_h[i]['inter_point'].add('head')
+#                     lines_v[j]['inter_point'].add('end')
+#                     is_per_h[i] = True
+#                     is_per_v[j] = True
+#                 elif abs(h['end'][0] - v['head'][0]) <= max_thickness:
+#                     lines_h[i]['inter_point'].add('end')
+#                     lines_v[j]['inter_point'].add('end')
+#                     is_per_h[i] = True
+#                     is_per_v[j] = True
+#     per_h = []
+#     per_v = []
+#     for i in range(len(is_per_h)):
+#         if is_per_h[i]:
+#             lines_h[i]['inter_point'] = list(lines_h[i]['inter_point'])
+#             per_h.append(lines_h[i])
+#     for i in range(len(is_per_v)):
+#         if is_per_v[i]:
+#             lines_v[i]['inter_point'] = list(lines_v[i]['inter_point'])
+#             per_v.append(lines_v[i])
+#     return per_h, per_v
+
+
+# def line_shrink_corners(corner, lines_h, lines_v):
+#     """
+#     shrink the corner according to lines:
+#              col_min_shrink: shrink right (increase)
+#              col_max_shrink: shrink left  (decrease)
+#              row_min_shrink: shrink down  (increase)
+#              row_max_shrink: shrink up    (decrease)
+#     :param lines_h: horizontal {'head':(column_min, row), 'end':(column_max, row), 'thickness':int)
+#     :param lines_v: vertical {'head':(column, row_min), 'end':(column, row_max), 'thickness':int}
+#     :return: shrunken corner: (top_left, bottom_right)
+#     """
+#     (col_min, row_min), (col_max, row_max) = corner
+#     col_min_shrink, row_min_shrink = col_min, row_min
+#     col_max_shrink, row_max_shrink = col_max, row_max
+#     valid_frame = False
+#
+#     for h in lines_h:
+#         # ignore outer border
+#         if len(h['inter_point']) == 2:
+#             valid_frame = True
+#             continue
+#         # shrink right -> col_min move to end
+#         if h['inter_point'][0] == 'head':
+#             col_min_shrink = max(h['end'][0], col_min_shrink)
+#         # shrink left -> col_max move to head
+#         elif h['inter_point'][0] == 'end':
+#             col_max_shrink = min(h['head'][0], col_max_shrink)
+#
+#     for v in lines_v:
+#         # ignore outer border
+#         if len(v['inter_point']) == 2:
+#             valid_frame = True
+#             continue
+#         # shrink down -> row_min move to end
+#         if v['inter_point'][0] == 'head':
+#             row_min_shrink = max(v['end'][1], row_min_shrink)
+#         # shrink up -> row_max move to head
+#         elif v['inter_point'][0] == 'end':
+#             row_max_shrink = min(v['head'][1], row_max_shrink)
+#
+#     # return the shrunken corner if only there is line intersecting with two other lines
+#     if valid_frame:
+#         return (col_min_shrink, row_min_shrink), (col_max_shrink, row_max_shrink)
+#     return corner
+
+
+# def line_cvt_relative_position(col_min, row_min, lines_h, lines_v):
+#     """
+#     convert the relative position of lines in the entire image
+#     :param col_min: based column the img lines belong to
+#     :param row_min: based row the img lines belong to
+#     :param lines_h: horizontal {'head':(column_min, row), 'end':(column_max, row), 'thickness':int)
+#     :param lines_v: vertical {'head':(column, row_min), 'end':(column, row_max), 'thickness':int}
+#     :return: lines_h_cvt, lines_v_cvt
+#     """
+#     for h in lines_h:
+#         h['head'][0] += col_min
+#         h['head'][1] += row_min
+#         h['end'][0] += col_min
+#         h['end'][1] += row_min
+#     for v in lines_v:
+#         v['head'][0] += col_min
+#         v['head'][1] += row_min
+#         v['end'][0] += col_min
+#         v['end'][1] += row_min
+#
+#     return lines_h, lines_v
+
+
+# check if an object is so slim
+# @boundary: [border_up, border_bottom, border_left, border_right]
+# -> up, bottom: (column_index, min/max row border)
+# -> left, right: (row_index, min/max column border) detect range of each row
+def clipping_by_line(boundary, boundary_rec, lines):
+    boundary = boundary.copy()
+    for orient in lines:
+        # horizontal
+        if orient == 'h':
+            # column range of sub area
+            r1, r2 = 0, 0
+            for line in lines[orient]:
+                if line[0] == 0:
+                    r1 = line[1]
+                    continue
+                r2 = line[0]
+                b_top = []
+                b_bottom = []
+                for i in range(len(boundary[0])):
+                    if r2 > boundary[0][i][0] >= r1:
+                        b_top.append(boundary[0][i])
+                for i in range(len(boundary[1])):
+                    if r2 > boundary[1][i][0] >= r1:
+                        b_bottom.append(boundary[1][i])
+
+                b_left = [x for x in boundary[2]]  # (row_index, min column border)
+                for i in range(len(b_left)):
+                    if b_left[i][1] < r1:
+                        b_left[i][1] = r1
+                b_right = [x for x in boundary[3]]  # (row_index, max column border)
+                for i in range(len(b_right)):
+                    if b_right[i][1] > r2:
+                        b_right[i][1] = r2
+
+                boundary_rec.append([b_top, b_bottom, b_left, b_right])
+                r1 = line[1]
+
+
+# remove imgs that contain text
+# def rm_text(org, corners, compo_class,
+#             max_text_height=C.THRESHOLD_TEXT_MAX_HEIGHT, max_text_width=C.THRESHOLD_TEXT_MAX_WIDTH,
+#             ocr_padding=C.OCR_PADDING, ocr_min_word_area=C.OCR_MIN_WORD_AREA, show=False):
+#     """
+#     Remove area that full of text
+#     :param org: original image
+#     :param corners: [(top_left, bottom_right)]
+#                     -> top_left: (column_min, row_min)
+#                     -> bottom_right: (column_max, row_max)
+#     :param compo_class: classes of corners
+#     :param max_text_height: Too large to be text
+#     :param max_text_width: Too large to be text
+#     :param ocr_padding: Padding for clipping
+#     :param ocr_min_word_area: If too text area ratio is too large
+#     :param show: Show or not
+#     :return: corners without text objects
+#     """
+#     new_corners = []
+#     new_class = []
+#     for i in range(len(corners)):
+#         corner = corners[i]
+#         (top_left, bottom_right) = corner
+#         (col_min, row_min) = top_left
+#         (col_max, row_max) = bottom_right
+#         height = row_max - row_min
+#         width = col_max - col_min
+#         # highly likely to be block or img if too large
+#         if height > max_text_height and width > max_text_width:
+#             new_corners.append(corner)
+#             new_class.append(compo_class[i])
+#         else:
+#             row_min = row_min - ocr_padding if row_min - ocr_padding >= 0 else 0
+#             row_max = row_max + ocr_padding if row_max + ocr_padding < org.shape[0] else org.shape[0]
+#             col_min = col_min - ocr_padding if col_min - ocr_padding >= 0 else 0
+#             col_max = col_max + ocr_padding if col_max + ocr_padding < org.shape[1] else org.shape[1]
+#             # check if this area is text
+#             clip = org[row_min: row_max, col_min: col_max]
+#             if not ocr.is_text(clip, ocr_min_word_area, show=show):
+#                 new_corners.append(corner)
+#                 new_class.append(compo_class[i])
+#     return new_corners, new_class
+
+
+# def rm_img_in_compo(corners_img, corners_compo):
+#     """
+#     Remove imgs in component
+#     """
+#     corners_img_new = []
+#     for img in corners_img:
+#         is_nested = False
+#         for compo in corners_compo:
+#             if util.corner_relation(img, compo) == -1:
+#                 is_nested = True
+#                 break
+#         if not is_nested:
+#             corners_img_new.append(img)
+#     return corners_img_new
+
+
+# def block_or_compo(org, binary, corners,
+#                    max_thickness=C.THRESHOLD_BLOCK_MAX_BORDER_THICKNESS, max_block_cross_points=C.THRESHOLD_BLOCK_MAX_CROSS_POINT,
+#                    min_compo_w_h_ratio=C.THRESHOLD_UICOMPO_MIN_W_H_RATIO, max_compo_w_h_ratio=C.THRESHOLD_UICOMPO_MAX_W_H_RATIO,
+#                    min_block_edge=C.THRESHOLD_BLOCK_MIN_EDGE_LENGTH):
+#     """
+#     Check if the objects are img components or just block
+#     :param org: Original image
+#     :param binary:  Binary image from pre-processing
+#     :param corners: [(top_left, bottom_right)]
+#                     -> top_left: (column_min, row_min)
+#                     -> bottom_right: (column_max, row_max)
+#     :param max_thickness: The max thickness of border of blocks
+#     :param max_block_cross_points: Ratio of point of interaction
+#     :return: corners of blocks and imgs
+#     """
+#     blocks = []
+#     imgs = []
+#     compos = []
+#     for corner in corners:
+#         (top_left, bottom_right) = corner
+#         (col_min, row_min) = top_left
+#         (col_max, row_max) = bottom_right
+#         height = row_max - row_min
+#         width = col_max - col_min
+#
+#         block = False
+#         vacancy = [0, 0, 0, 0]
+#         for i in range(1, max_thickness):
+#             try:
+#                 # top to bottom
+#                 if vacancy[0] == 0 and (col_max - col_min - 2 * i) is not 0 and (
+#                         np.sum(binary[row_min + i, col_min + i: col_max - i]) / 255) / (col_max - col_min - 2 * i) <= max_block_cross_points:
+#                     vacancy[0] = 1
+#                 # bottom to top
+#                 if vacancy[1] == 0 and (col_max - col_min - 2 * i) is not 0 and (
+#                         np.sum(binary[row_max - i, col_min + i: col_max - i]) / 255) / (col_max - col_min - 2 * i) <= max_block_cross_points:
+#                     vacancy[1] = 1
+#                 # left to right
+#                 if vacancy[2] == 0 and (row_max - row_min - 2 * i) is not 0 and (
+#                         np.sum(binary[row_min + i: row_max - i, col_min + i]) / 255) / (row_max - row_min - 2 * i) <= max_block_cross_points:
+#                     vacancy[2] = 1
+#                 # right to left
+#                 if vacancy[3] == 0 and (row_max - row_min - 2 * i) is not 0 and (
+#                         np.sum(binary[row_min + i: row_max - i, col_max - i]) / 255) / (row_max - row_min - 2 * i) <= max_block_cross_points:
+#                     vacancy[3] = 1
+#                 if np.sum(vacancy) == 4:
+#                     block = True
+#             except:
+#                 pass
+#
+#         # too big to be UI components
+#         if block:
+#             if height > min_block_edge and width > min_block_edge:
+#                 blocks.append(corner)
+#             else:
+#                 if min_compo_w_h_ratio < width / height < max_compo_w_h_ratio:
+#                     compos.append(corner)
+#         # filter out small objects
+#         else:
+#             if height > min_block_edge:
+#                 imgs.append(corner)
+#             else:
+#                 if min_compo_w_h_ratio < width / height < max_compo_w_h_ratio:
+#                     compos.append(corner)
+#     return blocks, imgs, compos
+
+
+# def compo_on_img(processing, org, binary, clf,
+#                  compos_corner, compos_class):
+#     """
+#     Detect potential UI components inner img;
+#     Only leave non-img
+#     """
+#     pad = 2
+#     for i in range(len(compos_corner)):
+#         if compos_class[i] != 'img':
+#             continue
+#         ((col_min, row_min), (col_max, row_max)) = compos_corner[i]
+#         col_min = max(col_min - pad, 0)
+#         col_max = min(col_max + pad, org.shape[1])
+#         row_min = max(row_min - pad, 0)
+#         row_max = min(row_max + pad, org.shape[0])
+#         area = (col_max - col_min) * (row_max - row_min)
+#         if area < 600:
+#             continue
+#
+#         clip_org = org[row_min:row_max, col_min:col_max]
+#         clip_bin_inv = pre.reverse_binary(binary[row_min:row_max, col_min:col_max])
+#
+#         compos_boundary_new, compos_corner_new, compos_class_new = processing(clip_org, clip_bin_inv, clf)
+#         compos_corner_new = util.corner_cvt_relative_position(compos_corner_new, col_min, row_min)
+#
+#         assert len(compos_corner_new) == len(compos_class_new)
+#
+#         # only leave non-img elements
+#         for i in range(len(compos_corner_new)):
+#             ((col_min_new, row_min_new), (col_max_new, row_max_new)) = compos_corner_new[i]
+#             area_new = (col_max_new - col_min_new) * (row_max_new - row_min_new)
+#             if compos_class_new[i] != 'img' and area_new / area < 0.8:
+#                 compos_corner.append(compos_corner_new[i])
+#                 compos_class.append(compos_class_new[i])
+#
+#     return compos_corner, compos_class
+
+
+# def strip_img(corners_compo, compos_class, corners_img):
+#     """
+#     Separate img from other compos
+#     :return: compos without img
+#     """
+#     corners_compo_withuot_img = []
+#     compo_class_withuot_img = []
+#     for i in range(len(compos_class)):
+#         if compos_class[i] == 'img':
+#             corners_img.append(corners_compo[i])
+#         else:
+#             corners_compo_withuot_img.append(corners_compo[i])
+#             compo_class_withuot_img.append(compos_class[i])
+#     return corners_compo_withuot_img, compo_class_withuot_img
+
+
+# def merge_corner(corners, compos_class, min_selected_IoU=C.THRESHOLD_MIN_IOU, is_merge_nested_same=True):
+#     """
+#     Calculate the Intersection over Overlap (IoU) and merge corners according to the value of IoU
+#     :param is_merge_nested_same: if true, merge the nested corners with same class whatever the IoU is
+#     :param corners: corners: [(top_left, bottom_right)]
+#                             -> top_left: (column_min, row_min)
+#                             -> bottom_right: (column_max, row_max)
+#     :return: new corners
+#     """
+#     new_corners = []
+#     new_class = []
+#     for i in range(len(corners)):
+#         is_intersected = False
+#         for j in range(len(new_corners)):
+#             r = util.corner_relation_nms(corners[i], new_corners[j], min_selected_IoU)
+#             # r = util.corner_relation(corners[i], new_corners[j])
+#             if is_merge_nested_same:
+#                 if compos_class[i] == new_class[j]:
+#                     # if corners[i] is in new_corners[j], ignore corners[i]
+#                     if r == -1:
+#                         is_intersected = True
+#                         break
+#                     # if new_corners[j] is in corners[i], replace new_corners[j] with corners[i]
+#                     elif r == 1:
+#                         is_intersected = True
+#                         new_corners[j] = corners[i]
+#
+#             # if above IoU threshold, and corners[i] is in new_corners[j], ignore corners[i]
+#             if r == -2:
+#                 is_intersected = True
+#                 break
+#             # if above IoU threshold, and new_corners[j] is in corners[i], replace new_corners[j] with corners[i]
+#             elif r == 2:
+#                 is_intersected = True
+#                 new_corners[j] = corners[i]
+#                 new_class[j] = compos_class[i]
+#
+#             # containing and too small
+#             elif r == -3:
+#                 is_intersected = True
+#                 break
+#             elif r == 3:
+#                 is_intersected = True
+#                 new_corners[j] = corners[i]
+#
+#             # if [i] and [j] are overlapped but no containing relation, merge corners when same class
+#             elif r == 4:
+#                 is_intersected = True
+#                 if compos_class[i] == new_class[j]:
+#                     new_corners[j] = util.corner_merge_two_corners(corners[i], new_corners[j])
+#
+#         if not is_intersected:
+#             new_corners.append(corners[i])
+#             new_class.append(compos_class[i])
+#     return new_corners, new_class
+
+
+# def select_corner(corners, compos_class, class_name):
+#     """
+#     Select corners in given compo type
+#     """
+#     corners_wanted = []
+#     for i in range(len(compos_class)):
+#         if compos_class[i] == class_name:
+#             corners_wanted.append(corners[i])
+#     return corners_wanted
+
+
+# def flood_fill_bfs(img, x_start, y_start, mark, grad_thresh):
+#     def neighbor(x, y):
+#         for i in range(x - 1, x + 2):
+#             if i < 0 or i >= img.shape[0]: continue
+#             for j in range(y - 1, y + 2):
+#                 if j < 0 or j >= img.shape[1]: continue
+#                 if mark[i, j] == 0 and abs(img[i, j] - img[x, y]) < grad_thresh:
+#                     stack.append([i, j])
+#                     mark[i, j] = 255
+#
+#     stack = [[x_start, y_start]]  # points waiting for inspection
+#     region = [[x_start, y_start]]  # points of this connected region
+#     mark[x_start, y_start] = 255  # drawing broad
+#     while len(stack) > 0:
+#         point = stack.pop()
+#         region.append(point)
+#         neighbor(point[0], point[1])
+#     return region

UIED/detect_compo/deprecated/ip_segment.py

@@ -0,0 +1,123 @@
+import cv2
+import numpy as np
+import shutil
+import os
+from os.path import join as pjoin
+
+
+def segment_img(org, segment_size, output_path, overlap=100):
+    if not os.path.exists(output_path):
+        os.mkdir(output_path)
+
+    height, width = np.shape(org)[0], np.shape(org)[1]
+    top = 0
+    bottom = segment_size
+    segment_no = 0
+    while top < height and bottom < height:
+        segment = org[top:bottom]
+        cv2.imwrite(os.path.join(output_path, str(segment_no) + '.png'), segment)
+        segment_no += 1
+        top += segment_size - overlap
+        bottom = bottom + segment_size - overlap if bottom + segment_size - overlap <= height else height
+
+
+def clipping(img, components, pad=0, show=False):
+    """
+    :param adjust: shrink(negative) or expand(positive) the bounding box
+    :param img: original image
+    :param corners: ((column_min, row_min),(column_max, row_max))
+    :return: list of clipping images
+    """
+    clips = []
+    for component in components:
+        clip = component.compo_clipping(img, pad=pad)
+        clips.append(clip)
+        if show:
+            cv2.imshow('clipping', clip)
+            cv2.waitKey()
+    return clips
+
+
+def dissemble_clip_img_hollow(clip_root, org, compos):
+    if os.path.exists(clip_root):
+        shutil.rmtree(clip_root)
+    os.mkdir(clip_root)
+    cls_dirs = []
+
+    bkg = org.copy()
+    hollow_out = np.ones(bkg.shape[:2], dtype=np.uint8) * 255
+    for compo in compos:
+        cls = compo.category
+        c_root = pjoin(clip_root, cls)
+        c_path = pjoin(c_root, str(compo.id) + '.jpg')
+        if cls not in cls_dirs:
+            os.mkdir(c_root)
+            cls_dirs.append(cls)
+        clip = compo.compo_clipping(org)
+        cv2.imwrite(c_path, clip)
+
+        col_min, row_min, col_max, row_max = compo.put_bbox()
+        hollow_out[row_min: row_max, col_min: col_max] = 0
+
+    bkg = cv2.merge((bkg, hollow_out))
+    cv2.imwrite(os.path.join(clip_root, 'bkg.png'), bkg)
+
+
+def dissemble_clip_img_fill(clip_root, org, compos, flag='most'):
+
+    def average_pix_around(pad=6, offset=3):
+        up = row_min - pad if row_min - pad >= 0 else 0
+        left = col_min - pad if col_min - pad >= 0 else 0
+        bottom = row_max + pad if row_max + pad < org.shape[0] - 1 else org.shape[0] - 1
+        right = col_max + pad if col_max + pad < org.shape[1] - 1 else org.shape[1] - 1
+
+        average = []
+        for i in range(3):
+            avg_up = np.average(org[up:row_min - offset, left:right, i])
+            avg_bot = np.average(org[row_max + offset:bottom, left:right, i])
+            avg_left = np.average(org[up:bottom, left:col_min - offset, i])
+            avg_right = np.average(org[up:bottom, col_max + offset:right, i])
+            average.append(int((avg_up + avg_bot + avg_left + avg_right)/4))
+        return average
+
+    def most_pix_around(pad=6, offset=2):
+        up = row_min - pad if row_min - pad >= 0 else 0
+        left = col_min - pad if col_min - pad >= 0 else 0
+        bottom = row_max + pad if row_max + pad < org.shape[0] - 1 else org.shape[0] - 1
+        right = col_max + pad if col_max + pad < org.shape[1] - 1 else org.shape[1] - 1
+
+        most = []
+        for i in range(3):
+            val = np.concatenate((org[up:row_min - offset, left:right, i].flatten(),
+                            org[row_max + offset:bottom, left:right, i].flatten(),
+                            org[up:bottom, left:col_min - offset, i].flatten(),
+                            org[up:bottom, col_max + offset:right, i].flatten()))
+            # print(val)
+            # print(np.argmax(np.bincount(val)))
+            most.append(int(np.argmax(np.bincount(val))))
+        return most
+
+    if os.path.exists(clip_root):
+        shutil.rmtree(clip_root)
+    os.mkdir(clip_root)
+    cls_dirs = []
+
+    bkg = org.copy()
+    for compo in compos:
+        cls = compo.category
+        c_root = pjoin(clip_root, cls)
+        c_path = pjoin(c_root, str(compo.id) + '.jpg')
+        if cls not in cls_dirs:
+            os.mkdir(c_root)
+            cls_dirs.append(cls)
+        clip = compo.compo_clipping(org)
+        cv2.imwrite(c_path, clip)
+
+        col_min, row_min, col_max, row_max = compo.put_bbox()
+        if flag == 'average':
+            color = average_pix_around()
+        elif flag == 'most':
+            color = most_pix_around()
+        cv2.rectangle(bkg, (col_min, row_min), (col_max, row_max), color, -1)
+
+    cv2.imwrite(os.path.join(clip_root, 'bkg.png'), bkg)

UIED/detect_compo/deprecated/ocr_classify_text.py

@@ -0,0 +1,113 @@
+import pytesseract as pyt
+import cv2
+
+import lib_ip.ip_draw as draw
+from config.CONFIG_UIED import Config
+
+C = Config()
+
+
+def is_text(img, min_word_area, show=False):
+    broad = img.copy()
+    area_word = 0
+    area_total = img.shape[0] * img.shape[1]
+
+    try:
+        # ocr text detection
+        data = pyt.image_to_data(img).split('\n')
+    except:
+        print(img.shape)
+        return -1
+    word = []
+    for d in data[1:]:
+        d = d.split()
+        if d[-1] != '-1':
+            if d[-1] != '-' and d[-1] != '—' and int(d[-3]) < 50 and int(d[-4]) < 100:
+                word.append(d)
+                t_l = (int(d[-6]), int(d[-5]))
+                b_r = (int(d[-6]) + int(d[-4]), int(d[-5]) + int(d[-3]))
+                area_word += int(d[-4]) * int(d[-3])
+                cv2.rectangle(broad, t_l, b_r, (0,0,255), 1)
+
+    if show:
+        for d in word: print(d)
+        print(area_word/area_total)
+        cv2.imshow('a', broad)
+        cv2.waitKey(0)
+        cv2.destroyAllWindows()
+    # no text in this clip or relatively small text area
+    if len(word) == 0 or area_word/area_total < min_word_area:
+        return False
+    return True
+
+
+def text_detection(org, img_clean):
+    try:
+        data = pyt.image_to_data(img_clean).split('\n')
+    except:
+        return org, None
+    corners_word = []
+    for d in data[1:]:
+        d = d.split()
+        if d[-1] != '-1':
+            if d[-1] != '-' and d[-1] != '—' and 5 < int(d[-3]) < 40 and 5 < int(d[-4]) < 100:
+                t_l = (int(d[-6]), int(d[-5]))
+                b_r = (int(d[-6]) + int(d[-4]), int(d[-5]) + int(d[-3]))
+                corners_word.append((t_l, b_r))
+    return corners_word
+
+
+# def text_merge_word_into_line(org, corners_word, max_words_gap=C.THRESHOLD_TEXT_MAX_WORD_GAP):
+#
+#     def is_in_line(word):
+#         for i in range(len(lines)):
+#             line = lines[i]
+#             # at the same row
+#             if abs(line['center'][1] - word['center'][1]) < max_words_gap:
+#                 # small gap between words
+#                 if (abs(line['center'][0] - word['center'][0]) - abs(line['width']/2 + word['width']/2)) < max_words_gap:
+#                     return i
+#         return -1
+#
+#     def merge_line(word, index):
+#         line = lines[index]
+#         # on the left
+#         if word['center'][0] < line['center'][0]:
+#             line['col_min'] = word['col_min']
+#         # on the right
+#         else:
+#             line['col_max'] = word['col_max']
+#         line['row_min'] = min(line['row_min'], word['row_min'])
+#         line['row_max'] = max(line['row_max'], word['row_max'])
+#         line['width'] = line['col_max'] - line['col_min']
+#         line['height'] = line['row_max'] - line['row_min']
+#         line['center'] = ((line['col_max'] + line['col_min'])/2, (line['row_max'] + line['row_min'])/2)
+#
+#     words = []
+#     for corner in corners_word:
+#         word = {}
+#         (top_left, bottom_right) = corner
+#         (col_min, row_min) = top_left
+#         (col_max, row_max) = bottom_right
+#         word['col_min'], word['col_max'], word['row_min'], word['row_max'] = col_min, col_max, row_min, row_max
+#         word['height'] = row_max - row_min
+#         word['width'] = col_max - col_min
+#         word['center'] = ((col_max + col_min)/2, (row_max + row_min)/2)
+#         words.append(word)
+#
+#     lines = []
+#     for word in words:
+#         line_index = is_in_line(word)
+#         # word is in current line
+#         if line_index != -1:
+#             merge_line(word, line_index)
+#         # word is not in current line
+#         else:
+#             # this single word as a new line
+#             lines.append(word)
+#
+#     corners_line = []
+#     for l in lines:
+#         corners_line.append(((l['col_min'], l['row_min']), (l['col_max'], l['row_max'])))
+#     return corners_line
+