{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "import numpy as np\n",
    "import cv2\n",
    "from glob import glob\n",
    "from os.path import join as pjoin\n",
    "from tqdm import tqdm\n",
    "\n",
    "\n",
    "def resize_label(bboxes, d_height, gt_height, bias=0):\n",
    "    bboxes_new = []\n",
    "    scale = gt_height / d_height\n",
    "    for bbox in bboxes:\n",
    "        bbox = [int(b * scale + bias) for b in bbox]\n",
    "        bboxes_new.append(bbox)\n",
    "    return bboxes_new\n",
    "\n",
    "\n",
    "def draw_bounding_box(org, corners, color=(0, 255, 0), line=2, show=False):\n",
    "    board = org.copy()\n",
    "    for i in range(len(corners)):\n",
    "        board = cv2.rectangle(board, (corners[i][0], corners[i][1]), (corners[i][2], corners[i][3]), color, line)\n",
    "    if show:\n",
    "        cv2.imshow('a', cv2.resize(board, (500, 1000)))\n",
    "        cv2.waitKey(0)\n",
    "    return board\n",
    "\n",
    "\n",
    "def load_detect_result_json(reslut_file_root, shrink=0):\n",
    "    def is_bottom_or_top(corner):\n",
    "        column_min, row_min, column_max, row_max = corner\n",
    "        if row_max < 36 or row_min > 725:\n",
    "            return True\n",
    "        return False\n",
    "\n",
    "    result_files = glob(pjoin(reslut_file_root, '*.json'))\n",
    "    compos_reform = {}\n",
    "    print('Loading %d detection results' % len(result_files))\n",
    "    for reslut_file in tqdm(result_files):\n",
    "        img_name = reslut_file.split('\\\\')[-1].split('.')[0]\n",
    "        compos = json.load(open(reslut_file, 'r'))['compos']\n",
    "        for compo in compos:\n",
    "            if is_bottom_or_top((compo['column_min'], compo['row_min'], compo['column_max'], compo['row_max'])):\n",
    "                continue\n",
    "            if img_name not in compos_reform:\n",
    "                compos_reform[img_name] = {'bboxes': [[compo['column_min'] + shrink, compo['row_min'] + shrink, compo['column_max'] - shrink, compo['row_max'] - shrink]],\n",
    "                                           'categories': [compo['category']]}\n",
    "            else:\n",
    "                compos_reform[img_name]['bboxes'].append([compo['column_min'] + shrink, compo['row_min'] + shrink, compo['column_max'] - shrink, compo['row_max'] - shrink])\n",
    "                compos_reform[img_name]['categories'].append(compo['category'])\n",
    "    return compos_reform\n",
    "\n",
    "\n",
    "def load_ground_truth_json(gt_file):\n",
    "    def get_img_by_id(img_id):\n",
    "        for image in images:\n",
    "            if image['id'] == img_id:\n",
    "                return image['file_name'].split('/')[-1][:-4], (image['height'], image['width'])\n",
    "\n",
    "    def cvt_bbox(bbox):\n",
    "        '''\n",
    "        :param bbox: [x,y,width,height]\n",
    "        :return: [col_min, row_min, col_max, row_max]\n",
    "        '''\n",
    "        bbox = [int(b) for b in bbox]\n",
    "        return [bbox[0], bbox[1], bbox[0] + bbox[2], bbox[1] + bbox[3]]\n",
    "\n",
    "    data = json.load(open(gt_file, 'r'))\n",
    "    images = data['images']\n",
    "    annots = data['annotations']\n",
    "    compos = {}\n",
    "    print('Loading %d ground truth' % len(annots))\n",
    "    for annot in tqdm(annots):\n",
    "        img_name, size = get_img_by_id(annot['image_id'])\n",
    "        if img_name not in compos:\n",
    "            compos[img_name] = {'bboxes': [cvt_bbox(annot['bbox'])], 'categories': [annot['category_id']], 'size': size}\n",
    "        else:\n",
    "            compos[img_name]['bboxes'].append(cvt_bbox(annot['bbox']))\n",
    "            compos[img_name]['categories'].append(annot['category_id'])\n",
    "    return compos\n",
    "\n",
    "\n",
    "def eval(detection, ground_truth, img_root, show=True, no_text=False, only_text=False):\n",
    "    def compo_filter(compos, flag):\n",
    "        if not no_text and not only_text:\n",
    "            return compos\n",
    "        compos_new = {'bboxes': [], 'categories': []}\n",
    "        for k, category in enumerate(compos['categories']):\n",
    "            if only_text:\n",
    "                if flag == 'det' and category != 'TextView':\n",
    "                    continue\n",
    "                if flag == 'gt' and int(category) != 14:\n",
    "                    continue\n",
    "            elif no_text:\n",
    "                if flag == 'det' and category == 'TextView':\n",
    "                    continue\n",
    "                if flag == 'gt' and int(category) == 14:\n",
    "                    continue\n",
    "\n",
    "            compos_new['bboxes'].append(compos['bboxes'][k])\n",
    "            compos_new['categories'].append(category)\n",
    "        return compos_new\n",
    "\n",
    "    def match(org, d_bbox, gt_bboxes, matched):\n",
    "        '''\n",
    "        :param matched: mark if the ground truth component is matched\n",
    "        :param d_bbox: [col_min, row_min, col_max, row_max]\n",
    "        :param gt_bboxes: list of ground truth [[col_min, row_min, col_max, row_max]]\n",
    "        :return: Boolean: if IOU large enough or detected box is contained by ground truth\n",
    "        '''\n",
    "        area_d = (d_bbox[2] - d_bbox[0]) * (d_bbox[3] - d_bbox[1])\n",
    "        for i, gt_bbox in enumerate(gt_bboxes):\n",
    "            if matched[i] == 0:\n",
    "                continue\n",
    "            area_gt = (gt_bbox[2] - gt_bbox[0]) * (gt_bbox[3] - gt_bbox[1])\n",
    "            col_min = max(d_bbox[0], gt_bbox[0])\n",
    "            row_min = max(d_bbox[1], gt_bbox[1])\n",
    "            col_max = min(d_bbox[2], gt_bbox[2])\n",
    "            row_max = min(d_bbox[3], gt_bbox[3])\n",
    "            # if not intersected, area intersection should be 0\n",
    "            w = max(0, col_max - col_min)\n",
    "            h = max(0, row_max - row_min)\n",
    "            area_inter = w * h\n",
    "            if area_inter == 0:\n",
    "                continue\n",
    "            iod = area_inter / area_d\n",
    "            iou = area_inter / (area_d + area_gt - area_inter)\n",
    "            # if show:\n",
    "            #     cv2.putText(org, (str(round(iou, 2)) + ',' + str(round(iod, 2))), (d_bbox[0], d_bbox[1]),\n",
    "            #                 cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 0, 255), 2)\n",
    "\n",
    "            if iou > 0.9 or iod == 1:\n",
    "                matched[i] = 0\n",
    "                return True\n",
    "        return False\n",
    "\n",
    "    amount = len(detection)\n",
    "    TP, FP, FN = 0, 0, 0\n",
    "    pres, recalls, f1s = [], [], []\n",
    "    for i, image_id in enumerate(detection):\n",
    "        TP_this, FP_this, FN_this = 0, 0, 0\n",
    "        img = cv2.imread(pjoin(img_root, image_id + '.jpg'))\n",
    "        d_compos = detection[image_id]\n",
    "        gt_compos = ground_truth[image_id]\n",
    "\n",
    "        org_height = gt_compos['size'][0]\n",
    "\n",
    "        d_compos = compo_filter(d_compos, 'det')\n",
    "        gt_compos = compo_filter(gt_compos, 'gt')\n",
    "\n",
    "        d_compos['bboxes'] = resize_label(d_compos['bboxes'], 800, org_height)\n",
    "        matched = np.ones(len(gt_compos['bboxes']), dtype=int)\n",
    "        for d_bbox in d_compos['bboxes']:\n",
    "            if match(img, d_bbox, gt_compos['bboxes'], matched):\n",
    "                TP += 1\n",
    "                TP_this += 1\n",
    "            else:\n",
    "                FP += 1\n",
    "                FP_this += 1\n",
    "        FN += sum(matched)\n",
    "        FN_this = sum(matched)\n",
    "\n",
    "        try:\n",
    "            pre_this = TP_this / (TP_this + FP_this)\n",
    "            recall_this = TP_this / (TP_this + FN_this)\n",
    "            f1_this = 2 * (pre_this * recall_this) / (pre_this + recall_this)\n",
    "        except:\n",
    "            print('empty')\n",
    "            continue\n",
    "\n",
    "        pres.append(pre_this)\n",
    "        recalls.append(recall_this)\n",
    "        f1s.append(f1_this)\n",
    "        if show:\n",
    "            print(image_id + '.jpg')\n",
    "            print('[%d/%d] TP:%d, FP:%d, FN:%d, Precesion:%.3f, Recall:%.3f' % (\n",
    "                i, amount, TP_this, FP_this, FN_this, pre_this, recall_this))\n",
    "            cv2.imshow('org', cv2.resize(img, (500, 1000)))\n",
    "            broad = draw_bounding_box(img, d_compos['bboxes'], color=(255, 0, 0), line=3)\n",
    "            draw_bounding_box(broad, gt_compos['bboxes'], color=(0, 0, 255), show=True, line=2)\n",
    "\n",
    "        if i % 200 == 0:\n",
    "            precision = TP / (TP + FP)\n",
    "            recall = TP / (TP + FN)\n",
    "            f1 = 2 * (precision * recall) / (precision + recall)\n",
    "            print(\n",
    "                '[%d/%d] TP:%d, FP:%d, FN:%d, Precesion:%.3f, Recall:%.3f, F1:%.3f' % (i, amount, TP, FP, FN, precision, recall, f1))\n",
    "\n",
    "    precision = TP / (TP + FP)\n",
    "    recall = TP / (TP + FN)\n",
    "    print('[%d/%d] TP:%d, FP:%d, FN:%d, Precesion:%.3f, Recall:%.3f, F1:%.3f' % (i, amount, TP, FP, FN, precision, recall, f1))\n",
    "    # print(\"Average precision:%.4f; Average recall:%.3f\" % (sum(pres)/len(pres), sum(recalls)/len(recalls)))\n",
    "\n",
    "    return pres, recalls, f1s"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import math\n",
    "\n",
    "def draw_plot(data, title='Score for our approach'):\n",
    "    for i in range(len(data)):\n",
    "        data[i] = [d for d in data[i] if not math.isnan(d)]\n",
    "#     plt.title(title)\n",
    "    labels = ['Precision', 'Recall', 'F1']\n",
    "    bplot = plt.boxplot(data, patch_artist=True, labels=labels)  # 设置箱型图可填充\n",
    "    colors = ['pink', 'lightblue', 'lightgreen']\n",
    "    for patch, color in zip(bplot['boxes'], colors):\n",
    "        patch.set_facecolor(color) \n",
    "    plt.grid(axis='y')\n",
    "    plt.xticks(fontsize=16)\n",
    "    plt.yticks(fontsize=16)\n",
    "    plt.savefig(title + '.png')\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  9%|███████▏                                                                     | 442/4708 [00:00<00:01, 4173.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading 4708 detection results\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████████████| 4708/4708 [00:01<00:00, 4404.67it/s]\n"
     ]
    }
   ],
   "source": [
    "detect = load_detect_result_json('E:\\\\Mulong\\\\Result\\\\rico\\\\rico_uied\\\\rico_new_uied_cls\\\\merge')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  8%|█████▉                                                                    | 6915/86646 [00:00<00:01, 68670.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading 86646 ground truth\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████████████████| 86646/86646 [00:11<00:00, 7576.11it/s]\n"
     ]
    }
   ],
   "source": [
    "gt = load_ground_truth_json('E:\\\\Mulong\\\\Datasets\\\\rico\\\\instances_test.json')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0/4707] TP:16, FP:0, FN:0, Precesion:1.000, Recall:1.000, F1:1.000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\Anaconda\\lib\\site-packages\\ipykernel_launcher.py:165: RuntimeWarning: invalid value encountered in double_scalars\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[200/4707] TP:2222, FP:2920, FN:1705, Precesion:0.432, Recall:0.566, F1:0.490\n",
      "[400/4707] TP:4616, FP:5737, FN:3346, Precesion:0.446, Recall:0.580, F1:0.504\n",
      "[600/4707] TP:6963, FP:8682, FN:4812, Precesion:0.445, Recall:0.591, F1:0.508\n",
      "[800/4707] TP:9367, FP:11432, FN:6305, Precesion:0.450, Recall:0.598, F1:0.514\n",
      "[1000/4707] TP:11222, FP:14346, FN:7511, Precesion:0.439, Recall:0.599, F1:0.507\n",
      "[1200/4707] TP:13680, FP:17278, FN:8901, Precesion:0.442, Recall:0.606, F1:0.511\n",
      "[1400/4707] TP:16274, FP:20664, FN:10379, Precesion:0.441, Recall:0.611, F1:0.512\n",
      "[1600/4707] TP:18431, FP:23002, FN:11556, Precesion:0.445, Recall:0.615, F1:0.516\n",
      "[1800/4707] TP:20718, FP:25600, FN:13049, Precesion:0.447, Recall:0.614, F1:0.517\n",
      "[2000/4707] TP:23009, FP:28626, FN:14588, Precesion:0.446, Recall:0.612, F1:0.516\n",
      "[2200/4707] TP:25424, FP:31555, FN:16191, Precesion:0.446, Recall:0.611, F1:0.516\n",
      "[2400/4707] TP:27559, FP:34176, FN:17388, Precesion:0.446, Recall:0.613, F1:0.517\n",
      "[2600/4707] TP:29820, FP:37065, FN:18617, Precesion:0.446, Recall:0.616, F1:0.517\n",
      "[2800/4707] TP:32108, FP:39846, FN:20018, Precesion:0.446, Recall:0.616, F1:0.518\n",
      "[3000/4707] TP:34188, FP:43112, FN:21399, Precesion:0.442, Recall:0.615, F1:0.515\n",
      "[3200/4707] TP:36558, FP:46011, FN:23002, Precesion:0.443, Recall:0.614, F1:0.514\n",
      "[3400/4707] TP:38783, FP:48918, FN:24365, Precesion:0.442, Recall:0.614, F1:0.514\n",
      "[3600/4707] TP:40958, FP:51829, FN:25605, Precesion:0.441, Recall:0.615, F1:0.514\n",
      "[3800/4707] TP:43270, FP:54963, FN:26841, Precesion:0.440, Recall:0.617, F1:0.514\n",
      "[4000/4707] TP:45512, FP:57838, FN:28141, Precesion:0.440, Recall:0.618, F1:0.514\n",
      "[4200/4707] TP:47544, FP:60789, FN:29420, Precesion:0.439, Recall:0.618, F1:0.513\n",
      "[4400/4707] TP:49907, FP:64407, FN:30897, Precesion:0.437, Recall:0.618, F1:0.512\n",
      "[4600/4707] TP:52181, FP:67592, FN:32399, Precesion:0.436, Recall:0.617, F1:0.511\n",
      "[4706/4707] TP:53393, FP:69230, FN:33248, Precesion:0.435, Recall:0.616, F1:0.511\n"
     ]
    }
   ],
   "source": [
    "no_text = False\n",
    "only_text = False\n",
    "pres_all, recalls_all, f1_all = eval(detect, gt, 'E:\\\\Mulong\\\\Datasets\\\\rico\\\\combined', show=False, no_text=no_text, only_text=only_text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0/4707] TP:1, FP:0, FN:0, Precesion:1.000, Recall:1.000, F1:1.000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\Anaconda\\lib\\site-packages\\ipykernel_launcher.py:165: RuntimeWarning: invalid value encountered in double_scalars\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[200/4707] TP:973, FP:2022, FN:891, Precesion:0.325, Recall:0.522, F1:0.400\n",
      "empty\n",
      "[400/4707] TP:1921, FP:3905, FN:1788, Precesion:0.330, Recall:0.518, F1:0.403\n",
      "[600/4707] TP:2847, FP:6079, FN:2717, Precesion:0.319, Recall:0.512, F1:0.393\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "[800/4707] TP:3774, FP:7895, FN:3574, Precesion:0.323, Recall:0.514, F1:0.397\n",
      "empty\n",
      "[1000/4707] TP:4478, FP:9951, FN:4229, Precesion:0.310, Recall:0.514, F1:0.387\n",
      "empty\n",
      "empty\n",
      "[1200/4707] TP:5451, FP:12055, FN:4960, Precesion:0.311, Recall:0.524, F1:0.391\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "[1400/4707] TP:6493, FP:14405, FN:5804, Precesion:0.311, Recall:0.528, F1:0.391\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "[1600/4707] TP:7372, FP:15980, FN:6375, Precesion:0.316, Recall:0.536, F1:0.397\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "[1800/4707] TP:8273, FP:17814, FN:7156, Precesion:0.317, Recall:0.536, F1:0.399\n",
      "empty\n",
      "empty\n",
      "[2000/4707] TP:9273, FP:19993, FN:8051, Precesion:0.317, Recall:0.535, F1:0.398\n",
      "empty\n",
      "[2200/4707] TP:10293, FP:22055, FN:8869, Precesion:0.318, Recall:0.537, F1:0.400\n",
      "[2400/4707] TP:11207, FP:23944, FN:9524, Precesion:0.319, Recall:0.541, F1:0.401\n",
      "empty\n",
      "empty\n",
      "[2600/4707] TP:12103, FP:25932, FN:10276, Precesion:0.318, Recall:0.541, F1:0.401\n",
      "[2800/4707] TP:12994, FP:27792, FN:11122, Precesion:0.319, Recall:0.539, F1:0.400\n",
      "empty\n",
      "empty\n",
      "[3000/4707] TP:13839, FP:30256, FN:11943, Precesion:0.314, Recall:0.537, F1:0.396\n",
      "[3200/4707] TP:14758, FP:32276, FN:12851, Precesion:0.314, Recall:0.535, F1:0.395\n",
      "empty\n",
      "[3400/4707] TP:15718, FP:34337, FN:13627, Precesion:0.314, Recall:0.536, F1:0.396\n",
      "[3600/4707] TP:16695, FP:36424, FN:14263, Precesion:0.314, Recall:0.539, F1:0.397\n",
      "[3800/4707] TP:17641, FP:38693, FN:14932, Precesion:0.313, Recall:0.542, F1:0.397\n",
      "empty\n",
      "empty\n",
      "[4000/4707] TP:18651, FP:40641, FN:15653, Precesion:0.315, Recall:0.544, F1:0.399\n",
      "empty\n",
      "[4200/4707] TP:19554, FP:42631, FN:16305, Precesion:0.314, Recall:0.545, F1:0.399\n",
      "empty\n",
      "empty\n",
      "[4400/4707] TP:20584, FP:45335, FN:17197, Precesion:0.312, Recall:0.545, F1:0.397\n",
      "[4600/4707] TP:21416, FP:47595, FN:17950, Precesion:0.310, Recall:0.544, F1:0.395\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "[4706/4707] TP:21870, FP:48657, FN:18391, Precesion:0.310, Recall:0.543, F1:0.395\n"
     ]
    }
   ],
   "source": [
    "no_text = True\n",
    "only_text = False\n",
    "pres_non_text, recalls_non_text, f1_non_text = eval(detect, gt, 'E:\\\\Mulong\\\\Datasets\\\\rico\\\\combined', show=False, no_text=no_text, only_text=only_text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0/4707] TP:15, FP:0, FN:0, Precesion:1.000, Recall:1.000, F1:1.000\n",
      "empty\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\Anaconda\\lib\\site-packages\\ipykernel_launcher.py:165: RuntimeWarning: invalid value encountered in double_scalars\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "[200/4707] TP:1041, FP:1106, FN:1022, Precesion:0.485, Recall:0.505, F1:0.495\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "[400/4707] TP:2185, FP:2342, FN:2068, Precesion:0.483, Recall:0.514, F1:0.498\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "[600/4707] TP:3272, FP:3447, FN:2939, Precesion:0.487, Recall:0.527, F1:0.506\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "[800/4707] TP:4505, FP:4625, FN:3819, Precesion:0.493, Recall:0.541, F1:0.516\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "[1000/4707] TP:5426, FP:5713, FN:4600, Precesion:0.487, Recall:0.541, F1:0.513\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "[1200/4707] TP:6649, FP:6803, FN:5521, Precesion:0.494, Recall:0.546, F1:0.519\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "[1400/4707] TP:7890, FP:8150, FN:6466, Precesion:0.492, Recall:0.550, F1:0.519\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "[1600/4707] TP:8964, FP:9117, FN:7276, Precesion:0.496, Recall:0.552, F1:0.522\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "[1800/4707] TP:10052, FP:10179, FN:8286, Precesion:0.497, Recall:0.548, F1:0.521\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "[2000/4707] TP:11126, FP:11243, FN:9147, Precesion:0.497, Recall:0.549, F1:0.522\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "[2200/4707] TP:12213, FP:12418, FN:10240, Precesion:0.496, Recall:0.544, F1:0.519\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "[2400/4707] TP:13243, FP:13341, FN:10973, Precesion:0.498, Recall:0.547, F1:0.521\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "[2600/4707] TP:14377, FP:14473, FN:11681, Precesion:0.498, Recall:0.552, F1:0.524\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "[2800/4707] TP:15494, FP:15674, FN:12516, Precesion:0.497, Recall:0.553, F1:0.524\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "[3000/4707] TP:16471, FP:16734, FN:13334, Precesion:0.496, Recall:0.553, F1:0.523\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "[3200/4707] TP:17644, FP:17891, FN:14307, Precesion:0.497, Recall:0.552, F1:0.523\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "[3400/4707] TP:18711, FP:18935, FN:15092, Precesion:0.497, Recall:0.554, F1:0.524\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "[3600/4707] TP:19710, FP:19958, FN:15895, Precesion:0.497, Recall:0.554, F1:0.524\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "[3800/4707] TP:20845, FP:21054, FN:16693, Precesion:0.498, Recall:0.555, F1:0.525\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "[4000/4707] TP:21881, FP:22177, FN:17468, Precesion:0.497, Recall:0.556, F1:0.525\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "[4200/4707] TP:22842, FP:23306, FN:18263, Precesion:0.495, Recall:0.556, F1:0.524\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "[4400/4707] TP:23930, FP:24465, FN:19093, Precesion:0.494, Recall:0.556, F1:0.524\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "[4600/4707] TP:25015, FP:25747, FN:20199, Precesion:0.493, Recall:0.553, F1:0.521\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "empty\n",
      "[4706/4707] TP:25638, FP:26458, FN:20742, Precesion:0.492, Recall:0.553, F1:0.521\n"
     ]
    }
   ],
   "source": [
    "no_text = False\n",
    "only_text = True\n",
    "pres_text, recalls_text, f1_text = eval(detect, gt, 'E:\\\\Mulong\\\\Datasets\\\\rico\\\\combined', show=False, no_text=no_text, only_text=only_text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\Anaconda\\lib\\site-packages\\matplotlib\\figure.py:448: UserWarning: Matplotlib is currently using module://ipykernel.pylab.backend_inline, which is a non-GUI backend, so cannot show the figure.\n",
      "  % get_backend())\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 5672 characters]\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "draw_plot([pres_all, recalls_all])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\Anaconda\\lib\\site-packages\\matplotlib\\figure.py:448: UserWarning: Matplotlib is currently using module://ipykernel.pylab.backend_inline, which is a non-GUI backend, so cannot show the figure.\n",
      "  % get_backend())\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 5636 characters]\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "draw_plot([pres_non_text, recalls_non_text])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "import pandas as pd\n",
    "\n",
    "pres1 = pd.DataFrame({'score_type':'Precision', 'score': pres_non_text, 'class':'Non_text'})\n",
    "pres2 = pd.DataFrame({'score_type':'Precision', 'score': pres_all, 'class':'All_element'})\n",
    "\n",
    "recalls1 = pd.DataFrame({'score_type':'Recall', 'score':recalls_non_text, 'class':'Non_text'})\n",
    "recalls2 = pd.DataFrame({'score_type':'Recall', 'score':recalls_all, 'class':'All_element'})\n",
    "\n",
    "f1s1 = pd.DataFrame({'score_type':'F1', 'score':f1_non_text, 'class':'Non_text'})\n",
    "f1s2 = pd.DataFrame({'score_type':'F1', 'score':f1_all, 'class':'All_element'})\n",
    "\n",
    "data=pd.concat([pres1, pres2, recalls1, recalls2, f1s1, f1s2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x179cafcdac8>"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 11328 characters]\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "sns.boxenplot(x='score_type', y='score', hue='class', data=data, width=0.5, linewidth=1.0, palette=\"Set3\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 8244 characters]\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "draw_plot([pres_all, recalls_all, f1_all], title='Scores for All Elements')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 8208 characters]\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "draw_plot([pres_non_text, recalls_non_text, f1_non_text], title='Score for Non-text Elements')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
