notebooks/analyse_predictions-confusion_contrastive_groups.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Annotation file paths"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "config_file_path = \"../configs/pretrained/config_model1.json\"\n",
    "contrastive_grp_file = \"../assets/contrastive_groups_list.txt\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import sys\n",
    "import pickle\n",
    "import numpy as np\n",
    "from pprint import pprint\n",
    "\n",
    "sys.path.insert(0, \"../\")\n",
    "\n",
    "from utils import load_json_config"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load config file\n",
    "config = load_json_config(config_file_path)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Get predictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "prediction_file_pickle_path = os.path.join('../', config['output_dir'], config['model_name'], 'test_results.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "## some weird hacky stuff\n",
    "with open(prediction_file_pickle_path, 'rb') as fp:\n",
    "    try:\n",
    "        logits_matrix, features_mat, targets_list, item_id_list, class_to_idx = pickle.load(fp)\n",
    "    except ValueError as e:\n",
    "        if str(e) != \"not enough values to unpack (expected 5, got 4)\":\n",
    "            raise\n",
    "        else:\n",
    "            with open(prediction_file_pickle_path, 'rb') as fp:\n",
    "                logits_matrix, targets_list, item_id_list, class_to_idx = pickle.load(fp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(24777, 174)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "logits_matrix.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(24777,)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "targets_list.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Fetch mapping"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "label_to_action_grp_dict = {}\n",
    "action_grp_to_label_dict = {}\n",
    "action_grp_to_target_dict = {}\n",
    "merge_grp_dict = {}\n",
    "\n",
    "grp_id = 0\n",
    "flag = 0\n",
    "\n",
    "with open(contrastive_grp_file, \"r\", encoding='utf-8') as fp:\n",
    "    for row in fp:\n",
    "        if row.startswith('# '):\n",
    "#             import pdb; pdb.set_trace()\n",
    "            if row[2].isdigit():\n",
    "                mapping = []\n",
    "                for c in row[2:].strip():\n",
    "                    if c.isdigit():\n",
    "                        mapping.append(int(c))\n",
    "                merge_grp_dict[grp_id] = mapping\n",
    "            continue\n",
    "        elif not row.strip():\n",
    "            if flag == 0:\n",
    "                flag = 1\n",
    "                continue\n",
    "            else:\n",
    "                flag = 0\n",
    "                grp_id += 1\n",
    "        elif row.startswith('##'):\n",
    "            break\n",
    "        else:\n",
    "            label = row.strip().strip(\",\").strip(\"\\\"\").strip(\"'\")\n",
    "            label_to_action_grp_dict[class_to_idx[label]] = grp_id\n",
    "            if grp_id not in action_grp_to_label_dict:\n",
    "                action_grp_to_label_dict[grp_id] = [label]\n",
    "                action_grp_to_target_dict[grp_id] = [class_to_idx[label]]                \n",
    "            else:                \n",
    "                action_grp_to_label_dict[grp_id].append(label)\n",
    "                action_grp_to_target_dict[grp_id].append(class_to_idx[label])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{24: [0, 0, 1, 1], 40: [0, 0, 1, 2], 67: [0, 0, 1]}"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "merge_grp_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 [173, 63]\n"
     ]
    }
   ],
   "source": [
    "for key, value in action_grp_to_target_dict.items():\n",
    "    print(key, value)\n",
    "    break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_argmax_over_predefined_targets(targets, logits):\n",
    "    return logits[targets].argmax()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Generate results for action groups separately (confusion mat, avg precision etc.)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# action_grp_preds_and_true = {}\n",
    "# for logits, target in zip(logits_matrix, targets_list):\n",
    "    \n",
    "#     ## if label is not present in action groups generated\n",
    "#     if target not in label_to_action_grp_dict:\n",
    "#         continue\n",
    "#     action_grp_belongingness = label_to_action_grp_dict[target]\n",
    "#     targets_action_grp = action_grp_to_target_dict[action_grp_belongingness]\n",
    "    \n",
    "#     pred_ag_label = get_argmax_over_predefined_targets(targets_action_grp, logits)\n",
    "#     true_ag_label = targets_action_grp.index(target)\n",
    "    \n",
    "#     if action_grp_belongingness not in action_grp_preds_and_true:\n",
    "#         action_grp_preds_and_true[action_grp_belongingness] = {\"y_pred\": [pred_ag_label],\n",
    "#                                                                \"y_true\": [true_ag_label]}\n",
    "#     else:\n",
    "#         action_grp_preds_and_true[action_grp_belongingness][\"y_pred\"].append(pred_ag_label)\n",
    "#         action_grp_preds_and_true[action_grp_belongingness][\"y_true\"].append(true_ag_label)    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "action_grp_preds_and_true = []\n",
    "for i in range(len(action_grp_to_target_dict)):\n",
    "    true_ag_label = []\n",
    "    pred_ag_label = []\n",
    "    \n",
    "    targets_action_grp = action_grp_to_target_dict[i]\n",
    "    for logits, target in zip(logits_matrix, targets_list):\n",
    "        if target not in targets_action_grp:\n",
    "            continue\n",
    "        else:\n",
    "            true_ag_label.append(targets_action_grp.index(target))\n",
    "            pred_ag_label.append(get_argmax_over_predefined_targets(targets_action_grp, logits))\n",
    "    \n",
    "    action_grp_preds_and_true.append({'y_true': true_ag_label, 'y_pred': pred_ag_label})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "def merge_classes(data, target_names, mapping_list):\n",
    "    mapping = {key: val for key, val in enumerate(mapping_list)}\n",
    "    \n",
    "    y_pred_new = []\n",
    "    for val in data['y_pred']:\n",
    "        y_pred_new.append(mapping[val])\n",
    "    \n",
    "    y_true_new = []\n",
    "    for val in data['y_true']:\n",
    "        y_true_new.append(mapping[val])\n",
    "    \n",
    "    data['y_pred'] = y_pred_new\n",
    "    data['y_true'] = y_true_new\n",
    "    \n",
    "    target_list_done = []\n",
    "    target_names_new = []\n",
    "    for i, elem in enumerate(mapping_list):\n",
    "        if elem not in target_list_done:\n",
    "            target_names_new.append(target_names[i])\n",
    "            target_list_done.append(elem)\n",
    "\n",
    "    return data, target_names_new"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import confusion_matrix\n",
    "from sklearn.metrics import classification_report\n",
    "from sklearn.metrics import precision_recall_fscore_support\n",
    "from sklearn.metrics import accuracy_score\n",
    "from pprint import pprint"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "################################################################################\n",
      "1: ACTION GROUP\n",
      "\n",
      "Confusion Matrix:\n",
      "[[119  18]\n",
      " [ 60  47]]\n",
      "0 --> Wiping something off of something\n",
      "1 --> Pretending or failing to wipe something off of something\n",
      "Average precision = 68.03% (56.15%)\n",
      "\n",
      "################################################################################\n",
      "2: ACTION GROUP\n",
      "\n",
      "Confusion Matrix:\n",
      "[[193  35]\n",
      " [ 49  57]]\n",
      "0 --> Closing something\n",
      "1 --> Pretending to close something without actually closing it\n",
      "Average precision = 74.85% (68.26%)\n",
      "\n",
      "################################################################################\n",
      "3: ACTION GROUP\n",
      "\n",
      "Confusion Matrix:\n",
      "[[256  76]\n",
      " [ 37 164]]\n",
      "0 --> Opening something\n",
      "1 --> Pretending to open something without actually opening it\n",
      "Average precision = 78.80% (62.29%)\n",
      "\n",
      "################################################################################\n",
      "4: ACTION GROUP\n",
      "\n",
      "Confusion Matrix:\n",
      "[[159  40]\n",
      " [  7 221]]\n",
      "0 --> Picking something up\n",
      "1 --> Pretending to pick something up\n",
      "Average precision = 88.99% (53.40%)\n",
      "\n",
      "################################################################################\n",
      "5: ACTION GROUP\n",
      "\n",
      "Confusion Matrix:\n",
      "[[366  25]\n",
      " [ 20  55]]\n",
      "0 --> Turning something upside down\n",
      "1 --> Pretending to turn something upside down\n",
      "Average precision = 90.34% (83.91%)\n",
      "\n",
      "################################################################################\n",
      "6: ACTION GROUP\n",
      "\n",
      "Confusion Matrix:\n",
      "[[275  17]\n",
      " [ 21  47]]\n",
      "0 --> Putting something into something\n",
      "1 --> Pretending to put something into something\n",
      "Average precision = 89.44% (81.11%)\n",
      "\n",
      "################################################################################\n",
      "7: ACTION GROUP\n",
      "\n",
      "Confusion Matrix:\n",
      "[[114  13]\n",
      " [ 10  13]]\n",
      "0 --> Putting something behind something\n",
      "1 --> Pretending to put something behind something\n",
      "Average precision = 84.67% (84.67%)\n",
      "\n",
      "################################################################################\n",
      "8: ACTION GROUP\n",
      "\n",
      "Confusion Matrix:\n",
      "[[192  11]\n",
      " [ 10  29]]\n",
      "0 --> Putting something next to something\n",
      "1 --> Pretending to put something next to something\n",
      "Average precision = 91.32% (83.88%)\n",
      "\n",
      "################################################################################\n",
      "9: ACTION GROUP\n",
      "\n",
      "Confusion Matrix:\n",
      "[[292   8]\n",
      " [ 11  95]]\n",
      "0 --> Putting something on a surface\n",
      "1 --> Pretending to put something on a surface\n",
      "Average precision = 95.32% (73.89%)\n",
      "\n",
      "################################################################################\n",
      "10: ACTION GROUP\n",
      "\n",
      "Confusion Matrix:\n",
      "[[136   3]\n",
      " [ 10  28]]\n",
      "0 --> Putting something onto something\n",
      "1 --> Pretending to put something onto something\n",
      "Average precision = 92.66% (78.53%)\n",
      "\n",
      "################################################################################\n",
      "11: ACTION GROUP\n",
      "\n",
      "Confusion Matrix:\n",
      "[[54  5]\n",
      " [ 5 13]]\n",
      "0 --> Putting something underneath something\n",
      "1 --> Pretending to put something underneath something\n",
      "Average precision = 87.01% (76.62%)\n",
      "\n",
      "################################################################################\n",
      "12: ACTION GROUP\n",
      "\n",
      "Confusion Matrix:\n",
      "[[54 11]\n",
      " [13 13]]\n",
      "0 --> Scooping something up with something\n",
      "1 --> Pretending to scoop something up with something\n",
      "Average precision = 73.63% (71.43%)\n",
      "\n",
      "################################################################################\n",
      "13: ACTION GROUP\n",
      "\n",
      "Confusion Matrix:\n",
      "[[169  15]\n",
      " [ 12  35]]\n",
      "0 --> Throwing something\n",
      "1 --> Pretending to throw something\n",
      "Average precision = 88.31% (79.65%)\n",
      "\n",
      "################################################################################\n",
      "14: ACTION GROUP\n",
      "\n",
      "Confusion Matrix:\n",
      "[[174  65]\n",
      " [ 11  36]]\n",
      "0 --> Taking something out of something\n",
      "1 --> Pretending to take something out of something\n",
      "Average precision = 73.43% (83.57%)\n",
      "\n",
      "################################################################################\n",
      "15: ACTION GROUP\n",
      "\n",
      "Confusion Matrix:\n",
      "[[59  5]\n",
      " [13 14]]\n",
      "0 --> Spreading something onto something\n",
      "1 --> Pretending to spread air onto something\n",
      "Average precision = 80.22% (70.33%)\n",
      "\n",
      "################################################################################\n",
      "16: ACTION GROUP\n",
      "\n",
      "Confusion Matrix:\n",
      "[[53 11]\n",
      " [11 47]]\n",
      "0 --> Sprinkling something onto something\n",
      "1 --> Pretending to sprinkle air onto something\n",
      "Average precision = 81.97% (52.46%)\n",
      "\n",
      "################################################################################\n",
      "17: ACTION GROUP\n",
      "\n",
      "Confusion Matrix:\n",
      "[[74  5]\n",
      " [ 4 52]]\n",
      "0 --> Pouring something out of something\n",
      "1 --> Pretending to pour something out of something, but something is empty\n",
      "Average precision = 93.33% (58.52%)\n",
      "\n",
      "################################################################################\n",
      "18: ACTION GROUP\n",
      "\n",
      "Confusion Matrix:\n",
      "[[110  18]\n",
      " [  6 119]]\n",
      "0 --> Taking something from somewhere\n",
      "1 --> Pretending to take something from somewhere\n",
      "Average precision = 90.51% (50.59%)\n",
      "\n",
      "################################################################################\n",
      "19: ACTION GROUP\n",
      "\n",
      "Confusion Matrix:\n",
      "[[ 60  15]\n",
      " [  5 199]]\n",
      "0 --> Twisting (wringing) something wet until water comes out\n",
      "1 --> Twisting something\n",
      "Average precision = 92.83% (73.12%)\n",
      "\n",
      "################################################################################\n",
      "20: ACTION GROUP\n",
      "\n",
      "Confusion Matrix:\n",
      "[[471  59]\n",
      " [ 37 331]]\n",
      "0 --> Tearing something into two pieces\n",
      "1 --> Tearing something just a little bit\n",
      "Average precision = 89.31% (59.02%)\n",
      "\n",
      "################################################################################\n",
      "21: ACTION GROUP\n",
      "\n",
      "Confusion Matrix:\n",
      "[[292  40]\n",
      " [ 21 207]]\n",
      "0 --> Opening something\n",
      "1 --> Closing something\n",
      "Average precision = 89.11% (59.29%)\n",
      "\n",
      "################################################################################\n",
      "22: ACTION GROUP\n",
      "\n",
      "Confusion Matrix:\n",
      "[[33  2]\n",
      " [ 1 23]]\n",
      "0 --> Poking a stack of something so the stack collapses\n",
      "1 --> Poking a stack of something without the stack collapsing\n",
      "Average precision = 94.92% (59.32%)\n",
      "\n",
      "################################################################################\n",
      "23: ACTION GROUP\n",
      "\n",
      "Confusion Matrix:\n",
      "[[141  11]\n",
      " [ 17  59]]\n",
      "0 --> Attaching something to something\n",
      "1 --> Trying but failing to attach something to something because it doesn't stick\n",
      "Average precision = 87.72% (66.67%)\n",
      "\n",
      "################################################################################\n",
      "24: ACTION GROUP\n",
      "\n",
      "Confusion Matrix:\n",
      "[[263  22]\n",
      " [ 40 175]]\n",
      "0 --> Folding something\n",
      "1 --> Unfolding something\n",
      "Average precision = 87.60% (57.00%)\n",
      "\n",
      "################################################################################\n",
      "25: ACTION GROUP\n",
      "\n",
      "Confusion Matrix:\n",
      "[[343  13]\n",
      " [ 11 249]]\n",
      "0 --> Moving something across a surface until it falls down\n",
      "1 --> Moving something across a surface without it falling down\n",
      "Average precision = 96.10% (57.79%)\n",
      "\n",
      "################################################################################\n",
      "26: ACTION GROUP\n",
      "\n",
      "Confusion Matrix:\n",
      "[[407  10]\n",
      " [ 12 300]]\n",
      "0 --> Covering something with something\n",
      "1 --> Uncovering something\n",
      "Average precision = 96.98% (57.20%)\n",
      "\n",
      "################################################################################\n",
      "27: ACTION GROUP\n",
      "\n",
      "Confusion Matrix:\n",
      "[[101   8]\n",
      " [ 14 113]]\n",
      "0 --> Throwing something in the air and catching it\n",
      "1 --> Throwing something in the air and letting it fall\n",
      "Average precision = 90.68% (53.81%)\n",
      "\n",
      "################################################################################\n",
      "28: ACTION GROUP\n",
      "\n",
      "Confusion Matrix:\n",
      "[[ 72   2]\n",
      " [  2 109]]\n",
      "0 --> Lifting up one end of something without letting it drop down\n",
      "1 --> Lifting up one end of something, then letting it drop down\n",
      "Average precision = 97.84% (60.00%)\n",
      "\n",
      "################################################################################\n",
      "29: ACTION GROUP\n",
      "\n",
      "Confusion Matrix:\n",
      "[[101   0]\n",
      " [  1 109]]\n",
      "0 --> Lifting something up completely without letting it drop down\n",
      "1 --> Lifting something up completely, then letting it drop down\n",
      "Average precision = 99.53% (52.13%)\n",
      "\n",
      "################################################################################\n",
      "30: ACTION GROUP\n",
      "\n",
      "Confusion Matrix:\n",
      "[[18  1]\n",
      " [ 3 37]]\n",
      "0 --> Lifting a surface with something on it but not enough for it to slide down\n",
      "1 --> Lifting a surface with something on it until it starts sliding down\n",
      "Average precision = 93.22% (67.80%)\n",
      "\n",
      "################################################################################\n",
      "31: ACTION GROUP\n",
      "\n",
      "Confusion Matrix:\n",
      "[[60  0]\n",
      " [10 47]]\n",
      "0 --> Pulling two ends of something so that it gets stretched\n",
      "1 --> Pulling two ends of something so that it separates into two pieces\n",
      "Average precision = 91.45% (51.28%)\n",
      "\n",
      "################################################################################\n",
      "32: ACTION GROUP\n",
      "\n",
      "Confusion Matrix:\n",
      "[[49  5]\n",
      " [ 3 31]]\n",
      "0 --> Moving something and something so they collide with each other\n",
      "1 --> Moving something and something so they pass each other\n",
      "Average precision = 90.91% (61.36%)\n",
      "\n",
      "################################################################################\n",
      "33: ACTION GROUP\n",
      "\n",
      "Confusion Matrix:\n",
      "[[316  52]\n",
      " [ 67 259]]\n",
      "0 --> Something falling like a feather or paper\n",
      "1 --> Something falling like a rock\n",
      "Average precision = 82.85% (53.03%)\n",
      "\n",
      "################################################################################\n",
      "34: ACTION GROUP\n",
      "\n",
      "Confusion Matrix:\n",
      "[[185  24]\n",
      " [ 25 233]]\n",
      "0 --> Spinning something so it continues spinning\n",
      "1 --> Spinning something that quickly stops spinning\n",
      "Average precision = 89.51% (55.25%)\n",
      "\n",
      "################################################################################\n",
      "35: ACTION GROUP\n",
      "\n",
      "Confusion Matrix:\n",
      "[[39  7]\n",
      " [ 1 53]]\n",
      "0 --> Putting something that can't roll onto a slanted surface, so it slides down\n",
      "1 --> Putting something that can't roll onto a slanted surface, so it stays where it is\n",
      "Average precision = 92.00% (54.00%)\n",
      "\n",
      "################################################################################\n",
      "36: ACTION GROUP\n",
      "\n",
      "Confusion Matrix:\n",
      "[[73 14]\n",
      " [10 47]]\n",
      "0 --> Something colliding with something and both are being deflected\n",
      "1 --> Something colliding with something and both come to a halt\n",
      "Average precision = 83.33% (60.42%)\n",
      "\n",
      "################################################################################\n",
      "37: ACTION GROUP\n",
      "\n",
      "Confusion Matrix:\n",
      "[[435  30]\n",
      " [ 82 155]]\n",
      "0 --> Plugging something into something\n",
      "1 --> Plugging something into something but pulling it right out as you remove your hand\n",
      "Average precision = 84.05% (66.24%)\n",
      "\n",
      "################################################################################\n",
      "38: ACTION GROUP\n",
      "\n",
      "Confusion Matrix:\n",
      "[[124   6]\n",
      " [  8 176]]\n",
      "0 --> Tilting something with something on it slightly so it doesn't fall down\n",
      "1 --> Tilting something with something on it until it falls off\n",
      "Average precision = 95.54% (58.60%)\n",
      "\n",
      "################################################################################\n",
      "39: ACTION GROUP\n",
      "\n",
      "Confusion Matrix:\n",
      "[[31  8]\n",
      " [ 4 29]]\n",
      "0 --> Burying something in something\n",
      "1 --> Digging something out of something\n",
      "Average precision = 83.33% (54.17%)\n",
      "\n",
      "################################################################################\n",
      "40: ACTION GROUP\n",
      "\n",
      "Confusion Matrix:\n",
      "[[ 1 14]\n",
      " [ 1 31]]\n",
      "0 --> Poking a hole into some substance\n",
      "1 --> Poking a hole into something soft\n",
      "Average precision = 68.09% (68.09%)\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "################################################################################\n",
      "41: ACTION GROUP\n",
      "\n",
      "Confusion Matrix:\n",
      "[[290   5   1]\n",
      " [ 10  95   1]\n",
      " [  5   3   9]]\n",
      "0 --> Poking something so it slightly moves\n",
      "1 --> Poking something so that it falls over\n",
      "2 --> Poking something so that it spins around\n",
      "Average precision = 94.03% (70.64%)\n",
      "\n",
      "################################################################################\n",
      "42: ACTION GROUP\n",
      "\n",
      "Confusion Matrix:\n",
      "[[276   2]\n",
      " [ 27  13]]\n",
      "0 --> Pouring something into something\n",
      "1 --> Trying to pour something into something, but missing so it spills next to it\n",
      "Average precision = 90.88% (87.42%)\n",
      "\n",
      "################################################################################\n",
      "43: ACTION GROUP\n",
      "\n",
      "Confusion Matrix:\n",
      "[[262   8   8]\n",
      " [ 19  30   1]\n",
      " [ 32   7  12]]\n",
      "0 --> Pouring something into something\n",
      "1 --> Pouring something into something until it overflows\n",
      "2 --> Pouring something onto something\n",
      "Average precision = 80.21% (73.35%)\n",
      "\n",
      "################################################################################\n",
      "44: ACTION GROUP\n",
      "\n",
      "Confusion Matrix:\n",
      "[[ 38   1]\n",
      " [ 10 407]]\n",
      "0 --> Burying something in something\n",
      "1 --> Covering something with something\n",
      "Average precision = 97.59% (91.45%)\n",
      "\n",
      "################################################################################\n",
      "45: ACTION GROUP\n",
      "\n",
      "Confusion Matrix:\n",
      "[[410   7]\n",
      " [  0  33]]\n",
      "0 --> Covering something with something\n",
      "1 --> Digging something out of something\n",
      "Average precision = 98.44% (92.67%)\n",
      "\n",
      "################################################################################\n",
      "46: ACTION GROUP\n",
      "\n",
      "Confusion Matrix:\n",
      "[[131  22]\n",
      " [ 30  73]]\n",
      "0 --> Bending something so that it deforms\n",
      "1 --> Bending something until it breaks\n",
      "Average precision = 79.69% (59.77%)\n",
      "\n",
      "################################################################################\n",
      "47: ACTION GROUP\n",
      "\n",
      "Confusion Matrix:\n",
      "[[116  11]\n",
      " [ 10 141]]\n",
      "0 --> Dropping something behind something\n",
      "1 --> Dropping something in front of something\n",
      "Average precision = 92.45% (54.32%)\n",
      "\n",
      "################################################################################\n",
      "48: ACTION GROUP\n",
      "\n",
      "Confusion Matrix:\n",
      "[[137  41]\n",
      " [ 40 194]]\n",
      "0 --> Dropping something into something\n",
      "1 --> Dropping something onto something\n",
      "Average precision = 80.34% (56.80%)\n",
      "\n",
      "################################################################################\n",
      "49: ACTION GROUP\n",
      "\n",
      "Confusion Matrix:\n",
      "[[117  10]\n",
      " [  3 124]]\n",
      "0 --> Dropping something behind something\n",
      "1 --> Putting something behind something\n",
      "Average precision = 94.88% (50.00%)\n",
      "\n",
      "################################################################################\n",
      "50: ACTION GROUP\n",
      "\n",
      "Confusion Matrix:\n",
      "[[147   4]\n",
      " [  6 129]]\n",
      "0 --> Dropping something in front of something\n",
      "1 --> Putting something in front of something\n",
      "Average precision = 96.50% (52.80%)\n",
      "\n",
      "################################################################################\n",
      "51: ACTION GROUP\n",
      "\n",
      "Confusion Matrix:\n",
      "[[159  19]\n",
      " [ 19 273]]\n",
      "0 --> Dropping something into something\n",
      "1 --> Putting something into something\n",
      "Average precision = 91.91% (62.13%)\n",
      "\n",
      "################################################################################\n",
      "52: ACTION GROUP\n",
      "\n",
      "Confusion Matrix:\n",
      "[[125  12]\n",
      " [  7 196]]\n",
      "0 --> Dropping something next to something\n",
      "1 --> Putting something next to something\n",
      "Average precision = 94.41% (59.71%)\n",
      "\n",
      "################################################################################\n",
      "53: ACTION GROUP\n",
      "\n",
      "Confusion Matrix:\n",
      "[[225   9]\n",
      " [  5 134]]\n",
      "0 --> Dropping something onto something\n",
      "1 --> Putting something onto something\n",
      "Average precision = 96.25% (62.73%)\n",
      "\n",
      "################################################################################\n",
      "54: ACTION GROUP\n",
      "\n",
      "Confusion Matrix:\n",
      "[[187  12]\n",
      " [  6 294]]\n",
      "0 --> Picking something up\n",
      "1 --> Putting something on a surface\n",
      "Average precision = 96.39% (60.12%)\n",
      "\n",
      "################################################################################\n",
      "55: ACTION GROUP\n",
      "\n",
      "Confusion Matrix:\n",
      "[[ 28  12]\n",
      " [ 12 280]]\n",
      "0 --> Failing to put something into something because something does not fit\n",
      "1 --> Putting something into something\n",
      "Average precision = 92.77% (87.95%)\n",
      "\n",
      "################################################################################\n",
      "56: ACTION GROUP\n",
      "\n",
      "Confusion Matrix:\n",
      "[[ 56   9]\n",
      " [ 11 188]]\n",
      "0 --> Scooping something up with something\n",
      "1 --> Picking something up\n",
      "Average precision = 92.42% (75.38%)\n",
      "\n",
      "################################################################################\n",
      "57: ACTION GROUP\n",
      "\n",
      "Confusion Matrix:\n",
      "[[209   9]\n",
      " [  3 106]]\n",
      "0 --> Tipping something over\n",
      "1 --> Touching (without moving) part of something\n",
      "Average precision = 96.33% (66.67%)\n",
      "\n",
      "################################################################################\n",
      "58: ACTION GROUP\n",
      "\n",
      "Confusion Matrix:\n",
      "[[203  15]\n",
      " [ 27  28]]\n",
      "0 --> Tipping something over\n",
      "1 --> Tipping something with something in it over, so something in it falls out\n",
      "Average precision = 84.62% (79.85%)\n",
      "\n",
      "################################################################################\n",
      "59: ACTION GROUP\n",
      "\n",
      "Confusion Matrix:\n",
      "[[ 55   0]\n",
      " [  1 108]]\n",
      "0 --> Tipping something with something in it over, so something in it falls out\n",
      "1 --> Touching (without moving) part of something\n",
      "Average precision = 99.39% (66.46%)\n",
      "\n",
      "################################################################################\n",
      "60: ACTION GROUP\n",
      "\n",
      "Confusion Matrix:\n",
      "[[ 96   0   4  11   5  11]\n",
      " [  3  96   5  12  11   8]\n",
      " [  3   2  37   4   4   9]\n",
      " [  5   2   6 172  10   8]\n",
      " [ 16   8   2  15  80  18]\n",
      " [ 14   2  10  10  23 233]]\n",
      "0 --> Putting something behind something\n",
      "1 --> Putting something in front of something\n",
      "2 --> Putting something underneath something\n",
      "3 --> Putting something next to something\n",
      "4 --> Putting something onto something\n",
      "5 --> Putting something into something\n",
      "Average precision = 74.76% (30.58%)\n",
      "\n",
      "################################################################################\n",
      "61: ACTION GROUP\n",
      "\n",
      "Confusion Matrix:\n",
      "[[274  14]\n",
      " [ 18 342]]\n",
      "0 --> Putting something similar to other things that are already on the table\n",
      "1 --> Taking one of many similar things on the table\n",
      "Average precision = 95.06% (55.56%)\n",
      "\n",
      "################################################################################\n",
      "62: ACTION GROUP\n",
      "\n",
      "Confusion Matrix:\n",
      "[[285   7]\n",
      " [ 24 215]]\n",
      "0 --> Putting something into something\n",
      "1 --> Taking something out of something\n",
      "Average precision = 94.16% (54.99%)\n",
      "\n",
      "################################################################################\n",
      "63: ACTION GROUP\n",
      "\n",
      "Confusion Matrix:\n",
      "[[288  12]\n",
      " [  7  64]]\n",
      "0 --> Putting something on a surface\n",
      "1 --> Putting something on the edge of something so it is not supported and falls down\n",
      "Average precision = 94.88% (80.86%)\n",
      "\n",
      "################################################################################\n",
      "64: ACTION GROUP\n",
      "\n",
      "Confusion Matrix:\n",
      "[[ 97   3  14]\n",
      " [  5 118   3]\n",
      " [ 15   3  83]]\n",
      "0 --> Laying something on the table on its side, not upright\n",
      "1 --> Putting something that cannot actually stand upright upright on the table, so it falls on its side\n",
      "2 --> Putting something upright on the table\n",
      "Average precision = 87.39% (36.95%)\n",
      "\n",
      "################################################################################\n",
      "65: ACTION GROUP\n",
      "\n",
      "Confusion Matrix:\n",
      "[[15  1  2  4  1]\n",
      " [ 0 30  2  3  4]\n",
      " [ 0  1 15  1  1]\n",
      " [11  2  2 50  3]\n",
      " [ 3  4  3 10 18]]\n",
      "0 --> Pretending to put something behind something\n",
      "1 --> Pretending to put something next to something\n",
      "2 --> Pretending to put something underneath something\n",
      "3 --> Pretending to put something into something\n",
      "4 --> Pretending to put something onto something\n",
      "Average precision = 68.82% (36.56%)\n",
      "\n",
      "################################################################################\n",
      "66: ACTION GROUP\n",
      "\n",
      "Confusion Matrix:\n",
      "[[ 70   4]\n",
      " [  3 200]]\n",
      "0 --> Putting something on a flat surface without letting it roll\n",
      "1 --> Rolling something on a flat surface\n",
      "Average precision = 97.47% (73.29%)\n",
      "\n",
      "################################################################################\n",
      "67: ACTION GROUP\n",
      "\n",
      "Confusion Matrix:\n",
      "[[ 8  1  8]\n",
      " [ 0 44 16]\n",
      " [ 4 12 93]]\n",
      "0 --> Spilling something behind something\n",
      "1 --> Spilling something next to something\n",
      "2 --> Spilling something onto something\n",
      "Average precision = 77.96% (58.60%)\n",
      "\n",
      "################################################################################\n",
      "68: ACTION GROUP\n",
      "\n",
      "Confusion Matrix:\n",
      "[[370   5]\n",
      " [  5  69]]\n",
      "0 --> Letting something roll along a flat surface\n",
      "1 --> Putting something on a flat surface without letting it roll\n",
      "Average precision = 97.77% (83.52%)\n",
      "\n",
      "################################################################################\n",
      "69: ACTION GROUP\n",
      "\n",
      "Confusion Matrix:\n",
      "[[15  3]\n",
      " [ 3 91]]\n",
      "0 --> Putting something onto a slanted surface but it doesn't glide down\n",
      "1 --> Letting something roll down a slanted surface\n",
      "Average precision = 94.64% (83.93%)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "action_grp_ap_average = 0\n",
    "action_grp_ap_most_probable_average = 0\n",
    "\n",
    "for action_grp, data in enumerate(action_grp_preds_and_true):\n",
    "    \n",
    "    target_names = []\n",
    "    for ind in action_grp_to_target_dict[action_grp]:\n",
    "        target_names.append(class_to_idx[ind])\n",
    "    \n",
    "    ## pre-process potential class clubbing\n",
    "    if action_grp in merge_grp_dict:\n",
    "        data, target_names = merge_classes(data, target_names, merge_grp_dict[action_grp])\n",
    "    \n",
    "\n",
    "    confusion_mat = confusion_matrix(data['y_true'], data['y_pred'])\n",
    "    report = classification_report(data['y_true'], data['y_pred'], target_names=target_names)\n",
    "\n",
    "    \n",
    "    \n",
    "    \"\"\"\n",
    "    Calculate metrics for each label, and find their average, weighted by support \n",
    "    (the number of true instances for each label). This alters ‘macro’ to account \n",
    "    for label imbalance; it can result in an F-score that is not between precision and recall.\n",
    "    \"\"\"\n",
    "    \n",
    "    precision_recall_fscore_support_val = precision_recall_fscore_support(\n",
    "                                                    data['y_true'],\n",
    "                                                    data['y_pred'],\n",
    "                                                    average='weighted'\n",
    "                                                    )\n",
    "    accuracy_group = accuracy_score(data['y_true'], data['y_pred'])\n",
    "    \n",
    "    print(\"#\" * 80)\n",
    "    print(\"{}: ACTION GROUP\".format(action_grp + 1))\n",
    "    print(\"\\nConfusion Matrix:\")\n",
    "    print(confusion_mat)\n",
    "    \n",
    "    for i, name in enumerate(target_names):\n",
    "        print(\"{} --> {}\".format(i, name))\n",
    "    \n",
    "#     print(report)\n",
    "    ap_most_probable = np.max(np.sum(confusion_mat, axis=1)) / np.sum(confusion_mat)\n",
    "    print(\"Average precision = {:.2f}% ({:.2f}%)\\n\".format(\n",
    "#                         precision_recall_fscore_support_val[0] * 100,\n",
    "                        accuracy_group * 100,\n",
    "                        ap_most_probable * 100))\n",
    "#     action_grp_ap_average += precision_recall_fscore_support_val[0]\n",
    "    action_grp_ap_average += accuracy_group\n",
    "    action_grp_ap_most_probable_average += ap_most_probable\n",
    "    \n",
    "action_grp_ap_average /= len(action_grp_preds_and_true)\n",
    "action_grp_ap_most_probable_average /= len(action_grp_preds_and_true)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Average precision among all action groups = 88.87%\n",
      "Average precision (most probable class) among all action groups= 64.93%\n"
     ]
    }
   ],
   "source": [
    "print(\"Average precision among all action groups = {:.2f}%\".format(action_grp_ap_average * 100))\n",
    "print(\"Average precision (most probable class) among all action groups= {:.2f}%\".format(action_grp_ap_most_probable_average * 100))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}