12_oov_eval.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 170,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The autoreload extension is already loaded. To reload it, use:\n",
      "  %reload_ext autoreload\n"
     ]
    }
   ],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 171,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-03-13T07:20:24.034028Z",
     "start_time": "2019-03-13T07:20:22.687626Z"
    }
   },
   "outputs": [],
   "source": [
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 172,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-03-13T07:20:24.382406Z",
     "start_time": "2019-03-13T07:20:24.037019Z"
    }
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 173,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-03-13T07:20:27.996727Z",
     "start_time": "2019-03-13T07:20:24.385088Z"
    }
   },
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "sns.set_context('paper')\n",
    "sns.set_style('white')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 174,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "sys.path.append('/home/nlp/danb')\n",
    "sys.path.append('/home/nlp/danb/NER')\n",
    "\n",
    "import bclm\n",
    "import ne_evaluate_mentions as nem"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Get all OOV words\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_folder = '../NER/data/for_ncrf'\n",
    "\n",
    "datasets = {\n",
    "    'morph': {\n",
    "        '_unit': 'morpheme',\n",
    "        '_scheme': 'bioes',\n",
    "        'train_dir': 'morph_gold_train.bmes',\n",
    "        'dev_dir': 'morph_gold_dev.bmes',\n",
    "        'test_dir': 'morph_gold_test.bmes', \n",
    "    },\n",
    "    'token': {\n",
    "        '_unit': 'token',\n",
    "        '_scheme': 'bioes',\n",
    "        'train_dir': 'token_gold_train_fix.bmes',\n",
    "        'dev_dir': 'token_gold_dev_fix.bmes',\n",
    "        'test_dir': 'token_gold_test_fix.bmes',\n",
    "    },\n",
    "    'multitok': {\n",
    "        '_unit': 'token',\n",
    "        '_scheme': 'concat_bioes',\n",
    "        'seg': False,\n",
    "        'train_dir': 'token_gold_train_concat.bmes',\n",
    "        'dev_dir': 'token_gold_dev_concat.bmes',\n",
    "        'test_dir': 'token_gold_test_concat.bmes',\n",
    "    },\n",
    "}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### out-of-training-vocabulary words (OOTV)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "../NER/data/for_ncrf/morph_gold_train.bmes\n",
      "../NER/data/for_ncrf/token_gold_train_fix.bmes\n",
      "../NER/data/for_ncrf/token_gold_train_concat.bmes\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[15959, 24889, 24889]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from collections import Counter\n",
    "\n",
    "train_words = {}\n",
    "for n, ds in datasets.items():\n",
    "    for k in ds:\n",
    "        if 'train' in k:\n",
    "            train_words[n] = []\n",
    "            path = os.path.join(data_folder, ds[k])\n",
    "            print(path)\n",
    "            for line in open(path, 'r'):\n",
    "                word = line.split(' ')[0].strip()\n",
    "                if word!='':\n",
    "                    train_words[n].append(word)\n",
    "            train_words[n] = set(train_words[n])\n",
    "                \n",
    "[len(x) for x in train_words.values()]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### out-of-embedding-vocabulary words (OOEV) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "word_embedding_files = {\n",
    "    'ft_yap': '../wordembedding-hebrew/vectors_alt_tok/wikipedia.alt_tok.yap_form.fasttext_skipgram.model.vec.nofirstline',\n",
    "    'ft_tok': '../wordembedding-hebrew/vectors_alt_tok/wikipedia.alt_tok.tokenized.fasttext_skipgram.model.vec.nofirstline',\n",
    "    'ft_oov_yap': 'data/htb_all_words.wikipedia.alt_tok.yap_form.fasttext_skipgram.txt',\n",
    "    'ft_oov_tok': 'data/htb_all_words.wikipedia.alt_tok.tokenized.fasttext_skipgram.txt',\n",
    "    #'w2v_yap': '../wordembedding-hebrew/vectors_alt_tok/wikipedia.alt_tok.yap_form.word2vec_skipgram.txt.nofirstline',\n",
    "    #'w2v_tok': '../wordembedding-hebrew/vectors_alt_tok/wikipedia.alt_tok.tokenized.word2vec_skipgram.txt.nofirstline',\n",
    "    'glv_yap': '../wordembedding-hebrew/vectors_alt_tok/wikipedia.alt_tok.yap_form.glove.txt',\n",
    "    'glv_tok': '../wordembedding-hebrew/vectors_alt_tok/wikipedia.alt_tok.tokenized.glove.txt',\n",
    "    'no_word': None,\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "emb_words = {}\n",
    "\n",
    "def get_words_from_emb_file(path):\n",
    "    words = []\n",
    "    for line in open(path):\n",
    "        line = line.strip()\n",
    "        if line!='':\n",
    "            words.append(line.split()[0])\n",
    "    return set(words)\n",
    "\n",
    "emb_words['tok'] = get_words_from_emb_file(word_embedding_files['glv_tok'])\n",
    "emb_words['yap'] = get_words_from_emb_file(word_embedding_files['glv_yap'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(359320, 245642)"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(emb_words['tok']), len(emb_words['yap'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "245641 ../wordembedding-hebrew/vectors_alt_tok/wikipedia.alt_tok.yap_form.fasttext_skipgram.model.vec.nofirstline\n",
      "359320 ../wordembedding-hebrew/vectors_alt_tok/wikipedia.alt_tok.tokenized.fasttext_skipgram.model.vec.nofirstline\n",
      "32852 data/htb_all_words.wikipedia.alt_tok.yap_form.fasttext_skipgram.txt\n",
      "32852 data/htb_all_words.wikipedia.alt_tok.tokenized.fasttext_skipgram.txt\n",
      "245642 ../wordembedding-hebrew/vectors_alt_tok/wikipedia.alt_tok.yap_form.glove.txt\n",
      "359320 ../wordembedding-hebrew/vectors_alt_tok/wikipedia.alt_tok.tokenized.glove.txt\n"
     ]
    }
   ],
   "source": [
    "for k, v in word_embedding_files.items():\n",
    "    if v is not None:\n",
    "        !wc -l {v}\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### IV (in-vocabulary)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "morph tok 14753\n",
      "morph yap 14926\n",
      "token tok 22487\n",
      "token yap 14175\n",
      "multitok tok 22487\n",
      "multitok yap 14175\n"
     ]
    }
   ],
   "source": [
    "inter = {}\n",
    "for ds, trw in train_words.items():\n",
    "    inter[ds] = {}\n",
    "    for eu, emw in emb_words.items():\n",
    "        inter[ds][eu] = trw & emw\n",
    "        print (ds, eu, len(inter[ds][eu]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### out-of-both-vocabulary words (OOBV)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "anything that isn't in the other three"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Find mentions that contain (parts of) unknown tokens"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "1. Token unknown (T)\n",
    "2. At least one morpheme unknown (M)\n",
    "3. At least one lemma unknown (L)\n",
    "4. All possible combinations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 175,
   "metadata": {},
   "outputs": [],
   "source": [
    "dropped = [5438, 5444, 5445, 5446, 5448, 5449, 5450, 5451, 5453, 5459]\n",
    "spdf = bclm.read_dataframe('spmrl')\n",
    "spdf = spdf[(~spdf.sent_id.isin(dropped))]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Full unknown\n",
    "Token unknown and at least one morpheme unknown (most obvious when token is made up of 1 morpheme)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 176,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_gold = spdf[spdf.set=='train']\n",
    "dev_gold = spdf[spdf.set=='dev']\n",
    "test_gold = spdf[spdf.set=='test']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 177,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_tokens = set(train_gold.token_str.tolist())\n",
    "dev_tokens =   set(dev_gold.token_str.tolist())\n",
    "test_tokens =  set(test_gold.token_str.tolist())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 178,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1500, 3995)"
      ]
     },
     "execution_count": 178,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dev_unk_tokens = dev_tokens-train_tokens\n",
    "test_unk_tokens = test_tokens-train_tokens\n",
    "len(dev_unk_tokens), len(dev_tokens)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 179,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((2738, 27), (11301, 27))"
      ]
     },
     "execution_count": 179,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dev_gold[~dev_gold.token_str.isin(train_tokens)].shape, dev_gold.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 180,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((1748,), (8531,))"
      ]
     },
     "execution_count": 180,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dev_gold[~dev_gold.token_str.isin(train_tokens)].groupby(['sent_id', 'token_id']).size().shape, dev_gold.groupby(['sent_id', 'token_id']).size().shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 181,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_morphs  = set(train_gold.form.tolist())\n",
    "dev_morphs    =   set(dev_gold.form.tolist())\n",
    "test_morphs   =  set(test_gold.form.tolist())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 182,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_lemmas  =   set(train_gold.lemma.tolist())\n",
    "dev_lemmas    =   set(dev_gold.lemma.tolist())\n",
    "test_lemmas  =    set(test_gold.lemma.tolist())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 183,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>form</th>\n",
       "      <th>lemma</th>\n",
       "      <th>upostag</th>\n",
       "      <th>xpostag</th>\n",
       "      <th>feats</th>\n",
       "      <th>token_id</th>\n",
       "      <th>sent_id</th>\n",
       "      <th>token_str</th>\n",
       "      <th>global_sent_id</th>\n",
       "      <th>...</th>\n",
       "      <th>deps</th>\n",
       "      <th>misc</th>\n",
       "      <th>ner_escaped</th>\n",
       "      <th>set</th>\n",
       "      <th>duplicate_sent_id</th>\n",
       "      <th>very_similar_sent_id</th>\n",
       "      <th>biose_layer0</th>\n",
       "      <th>biose_layer1</th>\n",
       "      <th>biose_layer2</th>\n",
       "      <th>biose_layer3</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>עשרות</td>\n",
       "      <td>עשר</td>\n",
       "      <td>CDT</td>\n",
       "      <td>CDT</td>\n",
       "      <td>gen=F|num=P</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>עשרות</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>_</td>\n",
       "      <td>_</td>\n",
       "      <td>_</td>\n",
       "      <td>dev</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>אנשים</td>\n",
       "      <td>איש</td>\n",
       "      <td>NN</td>\n",
       "      <td>NN</td>\n",
       "      <td>gen=M|num=P</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>אנשים</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>_</td>\n",
       "      <td>_</td>\n",
       "      <td>_</td>\n",
       "      <td>dev</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>מגיעים</td>\n",
       "      <td>הגיע</td>\n",
       "      <td>BN</td>\n",
       "      <td>BN</td>\n",
       "      <td>gen=M|num=P|per=A|HebBinyan=HIFIL</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>מגיעים</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>_</td>\n",
       "      <td>_</td>\n",
       "      <td>_</td>\n",
       "      <td>dev</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>מ</td>\n",
       "      <td>מ</td>\n",
       "      <td>PREPOSITION</td>\n",
       "      <td>PREPOSITION</td>\n",
       "      <td>_</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>מתאילנד</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>_</td>\n",
       "      <td>_</td>\n",
       "      <td>_</td>\n",
       "      <td>dev</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>תאילנד</td>\n",
       "      <td>תאילנד</td>\n",
       "      <td>NNP</td>\n",
       "      <td>NNP</td>\n",
       "      <td>_</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>מתאילנד</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>_</td>\n",
       "      <td>_</td>\n",
       "      <td>GPE</td>\n",
       "      <td>dev</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>S-GPE</td>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 27 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   id    form   lemma      upostag      xpostag  \\\n",
       "0   1   עשרות     עשר          CDT          CDT   \n",
       "1   2   אנשים     איש           NN           NN   \n",
       "2   3  מגיעים    הגיע           BN           BN   \n",
       "3   4       מ       מ  PREPOSITION  PREPOSITION   \n",
       "4   5  תאילנד  תאילנד          NNP          NNP   \n",
       "\n",
       "                               feats  token_id  sent_id token_str  \\\n",
       "0                        gen=F|num=P         1        1     עשרות   \n",
       "1                        gen=M|num=P         2        1     אנשים   \n",
       "2  gen=M|num=P|per=A|HebBinyan=HIFIL         3        1    מגיעים   \n",
       "3                                  _         4        1   מתאילנד   \n",
       "4                                  _         4        1   מתאילנד   \n",
       "\n",
       "   global_sent_id  ...  deps misc ner_escaped  set duplicate_sent_id  \\\n",
       "0               1  ...     _    _           _  dev               NaN   \n",
       "1               1  ...     _    _           _  dev               NaN   \n",
       "2               1  ...     _    _           _  dev               NaN   \n",
       "3               1  ...     _    _           _  dev               NaN   \n",
       "4               1  ...     _    _         GPE  dev               NaN   \n",
       "\n",
       "   very_similar_sent_id biose_layer0 biose_layer1 biose_layer2 biose_layer3  \n",
       "0                   NaN            O            O            O            O  \n",
       "1                   NaN            O            O            O            O  \n",
       "2                   NaN            O            O            O            O  \n",
       "3                   NaN            O            O            O            O  \n",
       "4                   NaN        S-GPE            O            O            O  \n",
       "\n",
       "[5 rows x 27 columns]"
      ]
     },
     "execution_count": 183,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dev_gold.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 184,
   "metadata": {},
   "outputs": [],
   "source": [
    "def unk_funq(x):\n",
    "    def get_unk_type(t, m, l):\n",
    "        ut = ''\n",
    "        if t:\n",
    "            ut+='T'\n",
    "        if m:\n",
    "            ut+='M'\n",
    "#        if l:\n",
    "#            ut+='L'\n",
    "        return ut\n",
    "    a = {}\n",
    "    a['token_unk'] = not (x.token_str.iat[0] in train_tokens)\n",
    "    a['morph_unk'] = (~x.form.isin(train_morphs)).any()\n",
    "    a['lemma_unk'] = (~x.lemma.isin(train_lemmas)).any()\n",
    "    a['unk_type'] = get_unk_type(a['token_unk'], a['morph_unk'], a['lemma_unk'])\n",
    "    a['has_ner'] = not (x.biose_layer0=='O').all()\n",
    "    a['morpheme_count'] = len(x)\n",
    "    a['token_id_rep'] = x.token_id.iat[0]\n",
    "\n",
    "    return pd.Series(a)\n",
    "\n",
    "dev_unk = (dev_gold.groupby(['sent_id', 'token_id', 'token_str'])\n",
    "                           .apply(unk_funq)).reset_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 185,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>unk_type</th>\n",
       "      <th>has_ner</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\"></th>\n",
       "      <th>False</th>\n",
       "      <td>6252</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>True</th>\n",
       "      <td>519</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>M</th>\n",
       "      <th>False</th>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">T</th>\n",
       "      <th>False</th>\n",
       "      <td>617</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>True</th>\n",
       "      <td>53</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">TM</th>\n",
       "      <th>False</th>\n",
       "      <td>820</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>True</th>\n",
       "      <td>258</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                     0\n",
       "unk_type has_ner      \n",
       "         False    6252\n",
       "         True      519\n",
       "M        False      12\n",
       "T        False     617\n",
       "         True       53\n",
       "TM       False     820\n",
       "         True      258"
      ]
     },
     "execution_count": 185,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dev_unk.groupby(['unk_type', 'has_ner']).size().to_frame()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 186,
   "metadata": {},
   "outputs": [],
   "source": [
    "def unk_funq(x):\n",
    "    def get_unk_type(t, m, l):\n",
    "        ut = ''\n",
    "        if t:\n",
    "            ut+='T'\n",
    "        if m:\n",
    "            ut+='M'\n",
    "#        if l:\n",
    "#            ut+='L'\n",
    "        return ut\n",
    "    a = {}\n",
    "    a['token_unk'] = not (x.token_str.iat[0] in train_tokens)\n",
    "    a['morph_unk'] = (~x.form.isin(train_morphs)).any()\n",
    "    a['lemma_unk'] = (~x.lemma.isin(train_lemmas)).any()\n",
    "    a['unk_type'] = get_unk_type(a['token_unk'], a['morph_unk'], a['lemma_unk'])\n",
    "    a['has_ner'] = not (x.biose_layer0=='O').all()\n",
    "    a['morpheme_count'] = len(x)\n",
    "    a['token_id_rep'] = x.token_id.iat[0]\n",
    "    return pd.Series(a)\n",
    "\n",
    "test_unk = (test_gold.groupby(['sent_id', 'token_id', 'token_str'])\n",
    "                           .apply(unk_funq)).reset_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 187,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>unk_type</th>\n",
       "      <th>has_ner</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\"></th>\n",
       "      <th>False</th>\n",
       "      <td>8746</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>True</th>\n",
       "      <td>940</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">M</th>\n",
       "      <th>False</th>\n",
       "      <td>38</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>True</th>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">T</th>\n",
       "      <th>False</th>\n",
       "      <td>944</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>True</th>\n",
       "      <td>102</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">TM</th>\n",
       "      <th>False</th>\n",
       "      <td>1389</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>True</th>\n",
       "      <td>458</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                     0\n",
       "unk_type has_ner      \n",
       "         False    8746\n",
       "         True      940\n",
       "M        False      38\n",
       "         True        2\n",
       "T        False     944\n",
       "         True      102\n",
       "TM       False    1389\n",
       "         True      458"
      ]
     },
     "execution_count": 187,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_unk.groupby(['unk_type', 'has_ner']).size().to_frame()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Create evalation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 188,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>gold_name</th>\n",
       "      <th>unit</th>\n",
       "      <th>arch</th>\n",
       "      <th>w_embed</th>\n",
       "      <th>seed_num</th>\n",
       "      <th>p_m</th>\n",
       "      <th>r_m</th>\n",
       "      <th>f_m</th>\n",
       "      <th>pred_set</th>\n",
       "      <th>acc</th>\n",
       "      <th>...</th>\n",
       "      <th>seg</th>\n",
       "      <th>model_file_name</th>\n",
       "      <th>dset_file_name</th>\n",
       "      <th>relevant_score</th>\n",
       "      <th>input_unit</th>\n",
       "      <th>embed_unit</th>\n",
       "      <th>embed_type</th>\n",
       "      <th>cm</th>\n",
       "      <th>pred_set_sub</th>\n",
       "      <th>pred_set_main</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>morph_dev_gold</td>\n",
       "      <td>morph</td>\n",
       "      <td>char_cnn</td>\n",
       "      <td>ft_yap</td>\n",
       "      <td>44_seed</td>\n",
       "      <td>0.859155</td>\n",
       "      <td>0.733467</td>\n",
       "      <td>0.791351</td>\n",
       "      <td>dev_gold</td>\n",
       "      <td>0.9667</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>morph.char_cnn.ft_yap.44_seed.83.model</td>\n",
       "      <td>morph.char_cnn.ft_yap.44_seed.dset</td>\n",
       "      <td>0.7905</td>\n",
       "      <td>morph</td>\n",
       "      <td>morph</td>\n",
       "      <td>ft</td>\n",
       "      <td>Match</td>\n",
       "      <td>gold</td>\n",
       "      <td>dev</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>morph_dev_yap</td>\n",
       "      <td>morph</td>\n",
       "      <td>char_cnn</td>\n",
       "      <td>ft_yap</td>\n",
       "      <td>44_seed</td>\n",
       "      <td>0.780193</td>\n",
       "      <td>0.647295</td>\n",
       "      <td>0.707558</td>\n",
       "      <td>dev_yap</td>\n",
       "      <td>0.9667</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>morph.char_cnn.ft_yap.44_seed.83.model</td>\n",
       "      <td>morph.char_cnn.ft_yap.44_seed.dset</td>\n",
       "      <td>0.7905</td>\n",
       "      <td>morph</td>\n",
       "      <td>morph</td>\n",
       "      <td>ft</td>\n",
       "      <td>Match</td>\n",
       "      <td>yap</td>\n",
       "      <td>dev</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>morph_test_gold</td>\n",
       "      <td>morph</td>\n",
       "      <td>char_cnn</td>\n",
       "      <td>ft_yap</td>\n",
       "      <td>44_seed</td>\n",
       "      <td>0.804850</td>\n",
       "      <td>0.747854</td>\n",
       "      <td>0.775306</td>\n",
       "      <td>test_gold</td>\n",
       "      <td>0.9667</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>morph.char_cnn.ft_yap.44_seed.83.model</td>\n",
       "      <td>morph.char_cnn.ft_yap.44_seed.dset</td>\n",
       "      <td>0.7905</td>\n",
       "      <td>morph</td>\n",
       "      <td>morph</td>\n",
       "      <td>ft</td>\n",
       "      <td>Match</td>\n",
       "      <td>gold</td>\n",
       "      <td>test</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>morph_dev_gold</td>\n",
       "      <td>morph</td>\n",
       "      <td>char_cnn</td>\n",
       "      <td>ft_oov_tok</td>\n",
       "      <td>44_seed</td>\n",
       "      <td>0.843612</td>\n",
       "      <td>0.767535</td>\n",
       "      <td>0.803778</td>\n",
       "      <td>dev_gold</td>\n",
       "      <td>0.9690</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>morph.char_cnn.ft_oov_tok.44_seed.183.model</td>\n",
       "      <td>morph.char_cnn.ft_oov_tok.44_seed.dset</td>\n",
       "      <td>0.8038</td>\n",
       "      <td>morph</td>\n",
       "      <td>token</td>\n",
       "      <td>ft_oov</td>\n",
       "      <td>Clash</td>\n",
       "      <td>gold</td>\n",
       "      <td>dev</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>morph_test_yap</td>\n",
       "      <td>morph</td>\n",
       "      <td>char_cnn</td>\n",
       "      <td>ft_yap</td>\n",
       "      <td>44_seed</td>\n",
       "      <td>0.721411</td>\n",
       "      <td>0.636266</td>\n",
       "      <td>0.676169</td>\n",
       "      <td>test_yap</td>\n",
       "      <td>0.9667</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>morph.char_cnn.ft_yap.44_seed.83.model</td>\n",
       "      <td>morph.char_cnn.ft_yap.44_seed.dset</td>\n",
       "      <td>0.7905</td>\n",
       "      <td>morph</td>\n",
       "      <td>morph</td>\n",
       "      <td>ft</td>\n",
       "      <td>Match</td>\n",
       "      <td>yap</td>\n",
       "      <td>test</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 56 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         gold_name   unit      arch     w_embed seed_num       p_m       r_m  \\\n",
       "0   morph_dev_gold  morph  char_cnn      ft_yap  44_seed  0.859155  0.733467   \n",
       "1    morph_dev_yap  morph  char_cnn      ft_yap  44_seed  0.780193  0.647295   \n",
       "2  morph_test_gold  morph  char_cnn      ft_yap  44_seed  0.804850  0.747854   \n",
       "3   morph_dev_gold  morph  char_cnn  ft_oov_tok  44_seed  0.843612  0.767535   \n",
       "4   morph_test_yap  morph  char_cnn      ft_yap  44_seed  0.721411  0.636266   \n",
       "\n",
       "        f_m   pred_set     acc  ...  seg  \\\n",
       "0  0.791351   dev_gold  0.9667  ...  NaN   \n",
       "1  0.707558    dev_yap  0.9667  ...  NaN   \n",
       "2  0.775306  test_gold  0.9667  ...  NaN   \n",
       "3  0.803778   dev_gold  0.9690  ...  NaN   \n",
       "4  0.676169   test_yap  0.9667  ...  NaN   \n",
       "\n",
       "                               model_file_name  \\\n",
       "0       morph.char_cnn.ft_yap.44_seed.83.model   \n",
       "1       morph.char_cnn.ft_yap.44_seed.83.model   \n",
       "2       morph.char_cnn.ft_yap.44_seed.83.model   \n",
       "3  morph.char_cnn.ft_oov_tok.44_seed.183.model   \n",
       "4       morph.char_cnn.ft_yap.44_seed.83.model   \n",
       "\n",
       "                           dset_file_name relevant_score input_unit  \\\n",
       "0      morph.char_cnn.ft_yap.44_seed.dset         0.7905      morph   \n",
       "1      morph.char_cnn.ft_yap.44_seed.dset         0.7905      morph   \n",
       "2      morph.char_cnn.ft_yap.44_seed.dset         0.7905      morph   \n",
       "3  morph.char_cnn.ft_oov_tok.44_seed.dset         0.8038      morph   \n",
       "4      morph.char_cnn.ft_yap.44_seed.dset         0.7905      morph   \n",
       "\n",
       "  embed_unit embed_type     cm pred_set_sub pred_set_main  \n",
       "0      morph         ft  Match         gold           dev  \n",
       "1      morph         ft  Match          yap           dev  \n",
       "2      morph         ft  Match         gold          test  \n",
       "3      token     ft_oov  Clash         gold           dev  \n",
       "4      morph         ft  Match          yap          test  \n",
       "\n",
       "[5 rows x 56 columns]"
      ]
     },
     "execution_count": 188,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mev = pd.read_pickle('final_setup/mev2.pkl')\n",
    "mev.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 698,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    [(עשרות, O), (אנשים, O), (מגיעים, O), (מ, O), ...\n",
       "1    [(תופעה, O), (זו, O), (התבררה, O), (אתמול, O),...\n",
       "2    [(יו\"ר, O), (ה, O), (וועדה, O), (,, O), (ח\"כ, ...\n",
       "3    [(מ, O), (צד, O), (אחד, O), (רוצה, O), (ה, B-O...\n",
       "4    [(נמיר, S-PER), (הודיעה, O), (כי, O), (תפנה, O...\n",
       "dtype: object"
      ]
     },
     "execution_count": 698,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "output_folder = 'final_setup/decode_output'\n",
    "pred_samp = nem.read_file_sents(os.path.join('final_setup/decode_output/morph_dev_gold.morph.char_cnn.ft_oov_tok.44_seed.bmes'))\n",
    "pred_samp.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 699,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sent_id</th>\n",
       "      <th>id</th>\n",
       "      <th>biose_layer0</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>S-GPE</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   sent_id  id biose_layer0\n",
       "0        1   1            O\n",
       "1        1   2            O\n",
       "2        1   3            O\n",
       "3        1   4            O\n",
       "4        1   5        S-GPE"
      ]
     },
     "execution_count": 699,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "for_df = []\n",
    "for i, line in pred_samp.iteritems():\n",
    "    sent_id=i+1\n",
    "    for j, (tok, bio) in enumerate(line):\n",
    "        for_df.append((sent_id, j+1, bio))\n",
    "ner_samp = pd.DataFrame(for_df, columns=['sent_id', 'id', 'biose_layer0'])\n",
    "ner_samp.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 700,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>form</th>\n",
       "      <th>lemma</th>\n",
       "      <th>upostag</th>\n",
       "      <th>xpostag</th>\n",
       "      <th>feats</th>\n",
       "      <th>head</th>\n",
       "      <th>deprel</th>\n",
       "      <th>deps</th>\n",
       "      <th>misc</th>\n",
       "      <th>sent_id</th>\n",
       "      <th>token_id</th>\n",
       "      <th>token_str</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>עשרות</td>\n",
       "      <td>עשר</td>\n",
       "      <td>CDT</td>\n",
       "      <td>CDT</td>\n",
       "      <td>gen=F|num=P</td>\n",
       "      <td>2</td>\n",
       "      <td>num</td>\n",
       "      <td>_</td>\n",
       "      <td>_</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>עשרות</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>אנשים</td>\n",
       "      <td>איש</td>\n",
       "      <td>NN</td>\n",
       "      <td>NN</td>\n",
       "      <td>gen=M|num=P</td>\n",
       "      <td>3</td>\n",
       "      <td>subj</td>\n",
       "      <td>_</td>\n",
       "      <td>_</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>אנשים</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>מגיעים</td>\n",
       "      <td>הגיע</td>\n",
       "      <td>BN</td>\n",
       "      <td>BN</td>\n",
       "      <td>gen=M|num=P|per=A</td>\n",
       "      <td>14</td>\n",
       "      <td>conj</td>\n",
       "      <td>_</td>\n",
       "      <td>_</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>מגיעים</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>מ</td>\n",
       "      <td>מ</td>\n",
       "      <td>PREPOSITION</td>\n",
       "      <td>PREPOSITION</td>\n",
       "      <td>NaN</td>\n",
       "      <td>3</td>\n",
       "      <td>comp</td>\n",
       "      <td>_</td>\n",
       "      <td>_</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>מתאילנד</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>תאילנד</td>\n",
       "      <td>תאילנד</td>\n",
       "      <td>NNP</td>\n",
       "      <td>NNP</td>\n",
       "      <td>gen=F|num=S</td>\n",
       "      <td>4</td>\n",
       "      <td>pobj</td>\n",
       "      <td>_</td>\n",
       "      <td>_</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>מתאילנד</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   id    form   lemma      upostag      xpostag              feats  head  \\\n",
       "0   1   עשרות     עשר          CDT          CDT        gen=F|num=P     2   \n",
       "1   2   אנשים     איש           NN           NN        gen=M|num=P     3   \n",
       "2   3  מגיעים    הגיע           BN           BN  gen=M|num=P|per=A    14   \n",
       "3   4       מ       מ  PREPOSITION  PREPOSITION                NaN     3   \n",
       "4   5  תאילנד  תאילנד          NNP          NNP        gen=F|num=S     4   \n",
       "\n",
       "  deprel deps misc  sent_id token_id token_str  \n",
       "0    num    _    _        1        1     עשרות  \n",
       "1   subj    _    _        1        2     אנשים  \n",
       "2   conj    _    _        1        3    מגיעים  \n",
       "3   comp    _    _        1        4   מתאילנד  \n",
       "4   pobj    _    _        1        4   מתאילנד  "
      ]
     },
     "execution_count": 700,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "yap_out = bclm.read_yap_output(treebank_set=None,\n",
    "                               tokens_path=bclm.TREEBANK_TOKEN_PATHS['dev'], \n",
    "                                dep_path='final_setup/pruned/yap_output/dev.multitok.char_cnn.ft_oov_tok.53_seed.conll',\n",
    "                                map_path='final_setup/pruned/yap_output/dev.multitok.char_cnn.ft_oov_tok.53_seed.map',)\n",
    "yap_out.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 701,
   "metadata": {},
   "outputs": [],
   "source": [
    "dev_gold = spdf[spdf.set=='dev']\n",
    "dev_gold = dev_gold.merge(dev_unk, how='left')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 702,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>form</th>\n",
       "      <th>lemma</th>\n",
       "      <th>upostag</th>\n",
       "      <th>xpostag</th>\n",
       "      <th>feats</th>\n",
       "      <th>head</th>\n",
       "      <th>deprel</th>\n",
       "      <th>deps</th>\n",
       "      <th>misc</th>\n",
       "      <th>sent_id</th>\n",
       "      <th>token_id</th>\n",
       "      <th>token_str</th>\n",
       "      <th>biose_layer0</th>\n",
       "      <th>token_unk</th>\n",
       "      <th>morph_unk</th>\n",
       "      <th>lemma_unk</th>\n",
       "      <th>unk_type</th>\n",
       "      <th>has_ner</th>\n",
       "      <th>morpheme_count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>11220</th>\n",
       "      <td>1</td>\n",
       "      <td>מתחיל</td>\n",
       "      <td>התחייל</td>\n",
       "      <td>BNT</td>\n",
       "      <td>BNT</td>\n",
       "      <td>gen=M|num=S|per=A</td>\n",
       "      <td>0</td>\n",
       "      <td>ROOT</td>\n",
       "      <td>_</td>\n",
       "      <td>_</td>\n",
       "      <td>500</td>\n",
       "      <td>1</td>\n",
       "      <td>מתחיל</td>\n",
       "      <td>O</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td></td>\n",
       "      <td>False</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11221</th>\n",
       "      <td>2</td>\n",
       "      <td>מסע</td>\n",
       "      <td>מסע</td>\n",
       "      <td>NNT</td>\n",
       "      <td>NNT</td>\n",
       "      <td>gen=M|num=S</td>\n",
       "      <td>1</td>\n",
       "      <td>gobj</td>\n",
       "      <td>_</td>\n",
       "      <td>_</td>\n",
       "      <td>500</td>\n",
       "      <td>2</td>\n",
       "      <td>מסע</td>\n",
       "      <td>O</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td></td>\n",
       "      <td>False</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11222</th>\n",
       "      <td>3</td>\n",
       "      <td>ה</td>\n",
       "      <td>ה</td>\n",
       "      <td>DEF</td>\n",
       "      <td>DEF</td>\n",
       "      <td>NaN</td>\n",
       "      <td>4</td>\n",
       "      <td>def</td>\n",
       "      <td>_</td>\n",
       "      <td>_</td>\n",
       "      <td>500</td>\n",
       "      <td>3</td>\n",
       "      <td>הנקמה</td>\n",
       "      <td>O</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td></td>\n",
       "      <td>False</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11223</th>\n",
       "      <td>4</td>\n",
       "      <td>נקמה</td>\n",
       "      <td>נקמה</td>\n",
       "      <td>NN</td>\n",
       "      <td>NN</td>\n",
       "      <td>gen=F|num=S</td>\n",
       "      <td>2</td>\n",
       "      <td>gobj</td>\n",
       "      <td>_</td>\n",
       "      <td>_</td>\n",
       "      <td>500</td>\n",
       "      <td>3</td>\n",
       "      <td>הנקמה</td>\n",
       "      <td>O</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td></td>\n",
       "      <td>False</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11224</th>\n",
       "      <td>5</td>\n",
       "      <td>.</td>\n",
       "      <td>NaN</td>\n",
       "      <td>yyDOT</td>\n",
       "      <td>yyDOT</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>punct</td>\n",
       "      <td>_</td>\n",
       "      <td>_</td>\n",
       "      <td>500</td>\n",
       "      <td>4</td>\n",
       "      <td>.</td>\n",
       "      <td>O</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td></td>\n",
       "      <td>False</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       id   form   lemma upostag xpostag              feats  head deprel deps  \\\n",
       "11220   1  מתחיל  התחייל     BNT     BNT  gen=M|num=S|per=A     0   ROOT    _   \n",
       "11221   2    מסע     מסע     NNT     NNT        gen=M|num=S     1   gobj    _   \n",
       "11222   3      ה       ה     DEF     DEF                NaN     4    def    _   \n",
       "11223   4   נקמה    נקמה      NN      NN        gen=F|num=S     2   gobj    _   \n",
       "11224   5      .     NaN   yyDOT   yyDOT                NaN     1  punct    _   \n",
       "\n",
       "      misc  sent_id token_id token_str biose_layer0  token_unk  morph_unk  \\\n",
       "11220    _      500        1     מתחיל            O      False      False   \n",
       "11221    _      500        2       מסע            O      False      False   \n",
       "11222    _      500        3     הנקמה            O      False      False   \n",
       "11223    _      500        3     הנקמה            O      False      False   \n",
       "11224    _      500        4         .            O      False      False   \n",
       "\n",
       "       lemma_unk unk_type  has_ner  morpheme_count  \n",
       "11220      False             False               1  \n",
       "11221      False             False               1  \n",
       "11222      False             False               2  \n",
       "11223      False             False               2  \n",
       "11224      False             False               1  "
      ]
     },
     "execution_count": 702,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "full_samp = yap_out.merge(ner_samp)\n",
    "full_samp = full_samp.merge(dev_unk, how='left')\n",
    "full_samp.tail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 703,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>form</th>\n",
       "      <th>lemma</th>\n",
       "      <th>upostag</th>\n",
       "      <th>xpostag</th>\n",
       "      <th>feats</th>\n",
       "      <th>token_id</th>\n",
       "      <th>sent_id</th>\n",
       "      <th>token_str</th>\n",
       "      <th>global_sent_id</th>\n",
       "      <th>...</th>\n",
       "      <th>biose_layer0</th>\n",
       "      <th>biose_layer1</th>\n",
       "      <th>biose_layer2</th>\n",
       "      <th>biose_layer3</th>\n",
       "      <th>token_unk</th>\n",
       "      <th>morph_unk</th>\n",
       "      <th>lemma_unk</th>\n",
       "      <th>unk_type</th>\n",
       "      <th>has_ner</th>\n",
       "      <th>morpheme_count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>11296</th>\n",
       "      <td>1</td>\n",
       "      <td>מתחיל</td>\n",
       "      <td>התחיל</td>\n",
       "      <td>BN</td>\n",
       "      <td>BN</td>\n",
       "      <td>gen=M|num=S|per=A</td>\n",
       "      <td>1</td>\n",
       "      <td>500</td>\n",
       "      <td>מתחיל</td>\n",
       "      <td>500</td>\n",
       "      <td>...</td>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td></td>\n",
       "      <td>False</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11297</th>\n",
       "      <td>2</td>\n",
       "      <td>מסע</td>\n",
       "      <td>מסע</td>\n",
       "      <td>NNT</td>\n",
       "      <td>NNT</td>\n",
       "      <td>gen=M|num=S</td>\n",
       "      <td>2</td>\n",
       "      <td>500</td>\n",
       "      <td>מסע</td>\n",
       "      <td>500</td>\n",
       "      <td>...</td>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td></td>\n",
       "      <td>False</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11298</th>\n",
       "      <td>3</td>\n",
       "      <td>ה</td>\n",
       "      <td>ה</td>\n",
       "      <td>DEF</td>\n",
       "      <td>DEF</td>\n",
       "      <td>_</td>\n",
       "      <td>3</td>\n",
       "      <td>500</td>\n",
       "      <td>הנקמה</td>\n",
       "      <td>500</td>\n",
       "      <td>...</td>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td></td>\n",
       "      <td>False</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11299</th>\n",
       "      <td>4</td>\n",
       "      <td>נקמה</td>\n",
       "      <td>נקמה</td>\n",
       "      <td>NN</td>\n",
       "      <td>NN</td>\n",
       "      <td>gen=F|num=S</td>\n",
       "      <td>3</td>\n",
       "      <td>500</td>\n",
       "      <td>הנקמה</td>\n",
       "      <td>500</td>\n",
       "      <td>...</td>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td></td>\n",
       "      <td>False</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11300</th>\n",
       "      <td>5</td>\n",
       "      <td>.</td>\n",
       "      <td>_</td>\n",
       "      <td>yyDOT</td>\n",
       "      <td>yyDOT</td>\n",
       "      <td>_</td>\n",
       "      <td>4</td>\n",
       "      <td>500</td>\n",
       "      <td>.</td>\n",
       "      <td>500</td>\n",
       "      <td>...</td>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td></td>\n",
       "      <td>False</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 33 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       id   form  lemma upostag xpostag              feats  token_id  sent_id  \\\n",
       "11296   1  מתחיל  התחיל      BN      BN  gen=M|num=S|per=A         1      500   \n",
       "11297   2    מסע    מסע     NNT     NNT        gen=M|num=S         2      500   \n",
       "11298   3      ה      ה     DEF     DEF                  _         3      500   \n",
       "11299   4   נקמה   נקמה      NN      NN        gen=F|num=S         3      500   \n",
       "11300   5      .      _   yyDOT   yyDOT                  _         4      500   \n",
       "\n",
       "      token_str  global_sent_id  ...  biose_layer0 biose_layer1 biose_layer2  \\\n",
       "11296     מתחיל             500  ...             O            O            O   \n",
       "11297       מסע             500  ...             O            O            O   \n",
       "11298     הנקמה             500  ...             O            O            O   \n",
       "11299     הנקמה             500  ...             O            O            O   \n",
       "11300         .             500  ...             O            O            O   \n",
       "\n",
       "       biose_layer3 token_unk  morph_unk lemma_unk unk_type has_ner  \\\n",
       "11296             O     False      False     False            False   \n",
       "11297             O     False      False     False            False   \n",
       "11298             O     False      False     False            False   \n",
       "11299             O     False      False     False            False   \n",
       "11300             O     False      False     False            False   \n",
       "\n",
       "      morpheme_count  \n",
       "11296              1  \n",
       "11297              1  \n",
       "11298              2  \n",
       "11299              2  \n",
       "11300              1  \n",
       "\n",
       "[5 rows x 33 columns]"
      ]
     },
     "execution_count": 703,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "full_samp = dev_gold.copy()\n",
    "full_samp['biose_layer0'] = ner_samp['biose_layer0']\n",
    "full_samp = full_samp.merge(dev_unk, how='left')\n",
    "full_samp.tail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 704,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((11301, 33), (11301, 13), (11301, 3))"
      ]
     },
     "execution_count": 704,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "full_samp.shape, yap_out.shape, ner_samp.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 678,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1, 2, 4, 5)"
      ]
     },
     "execution_count": 678,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tuple(sorted(set([1,4,1,5,2])))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Grouped mention evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "from collections import defaultdict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "def ut_gm(x):\n",
    "    filt_x = list(filter(lambda a: a != '', x))\n",
    "    return tuple(sorted(set(filt_x)))\n",
    "    \n",
    "def sent_to_mentions_dict(sent, sent_id, truncate=None, group_maker=ut_gm, ment_len=False,\n",
    "                         ment_cat=False):\n",
    "    mentions = defaultdict(lambda: defaultdict(lambda: 0))\n",
    "    current_mention= None\n",
    "    current_cat = None\n",
    "    current_group = []\n",
    "    if truncate is not None:\n",
    "        it = islice(sent, truncate)\n",
    "    else:\n",
    "        it = sent\n",
    "    \n",
    "    for tok, bio, cat, group in it:\n",
    "        if bio=='S':\n",
    "            current_group = [group]\n",
    "            final_group = group_maker(current_group)\n",
    "            if ment_len:\n",
    "                if final_group is None:\n",
    "                    final_group = 1\n",
    "                else:\n",
    "                    final_group = (final_group, 1)\n",
    "            if ment_cat:\n",
    "                final_group = (final_group, cat)\n",
    "            mentions[final_group][(sent_id, tok, cat)]+=1\n",
    "            current_mention= None\n",
    "            current_cat = None\n",
    "            current_group = []\n",
    "\n",
    "        if bio=='B':\n",
    "            current_mention = [tok]\n",
    "            current_cat = cat\n",
    "            current_group.append(group)\n",
    "        if bio=='I' and current_mention is not None:\n",
    "            current_mention.append(tok)\n",
    "            current_group.append(group)\n",
    "        if bio=='E' and current_mention is not None:\n",
    "            current_mention.append(tok)\n",
    "            current_group.append(group)\n",
    "            final_group = group_maker(current_group)\n",
    "            if ment_len:\n",
    "                if final_group is None:\n",
    "                    final_group = len(current_mention)\n",
    "                else:\n",
    "                    final_group = (final_group, len(current_mention))\n",
    "                if ment_cat:\n",
    "                    final_group = (final_group, current_cat)\n",
    "            mentions[final_group][(sent_id, ' '.join(current_mention), current_cat)]+=1\n",
    "            current_mention = None\n",
    "            current_cat = None\n",
    "            current_group = []\n",
    "        if bio=='O':\n",
    "            current_mention = None\n",
    "            current_cat = None\n",
    "            current_group = []\n",
    "\n",
    "    return mentions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_sent(g, cols):\n",
    "    sent = []\n",
    "    for i, r in g.iterrows():\n",
    "        sent.append(tuple(r[cols].tolist()))\n",
    "    return sent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_sents_fixed(sents):\n",
    "    sf = []\n",
    "    for sent in sents:\n",
    "        new_sent = []\n",
    "        for tok, biose, *group in sent:\n",
    "            if (len(group)==1):\n",
    "                group = group[0]\n",
    "            else:\n",
    "                group = tuple(group)\n",
    "            tag = biose.split('-')\n",
    "            biose = tag[0]\n",
    "            if len(tag)>1:\n",
    "                cat = tag[1]\n",
    "            else:\n",
    "                cat = '_'\n",
    "            new_sent.append((tok, biose, cat, group))\n",
    "        sf.append(new_sent)\n",
    "    sf = list(zip(list(sents.index), sf))\n",
    "    return sf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_ment_set(sents):\n",
    "    ment_set = defaultdict(list)\n",
    "    for sent in sents:\n",
    "        for group, gment in sent.items():\n",
    "            for k, val in gment.items():\n",
    "                for i in range(val):\n",
    "                    ment_set[group].append((k[0], k[1], k[2], i+1))\n",
    "    return ment_set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 230,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict_keys([('T',), (), ('TM',), ('T', 'TM')])"
      ]
     },
     "execution_count": 230,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "token_unk_sents = full_samp.groupby('sent_id').apply(get_sent, ['form', 'biose_layer0', 'unk_type'])\n",
    "token_unk_sents_fixed = get_sents_fixed(token_unk_sents)\n",
    "ments = [sent_to_mentions_dict(sent, sent_id) for sent_id, sent in token_unk_sents_fixed]\n",
    "ment_set = get_ment_set(ments)\n",
    "ment_set.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 231,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict_keys([('T',), (), ('TM',), ('T', 'TM')])"
      ]
     },
     "execution_count": 231,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gold_dev_sents = dev_gold.groupby('sent_id').apply(get_sent, ['form', 'biose_layer0', 'unk_type'])\n",
    "gold_dev_sents_fixed = get_sents_fixed(gold_dev_sents)\n",
    "gold_ments = [sent_to_mentions_dict(sent, sent_id) for sent_id, sent in gold_dev_sents_fixed]\n",
    "gold_ment_set = get_ment_set(gold_ments)\n",
    "gold_ment_set.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 232,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{(), ('T',), ('T', 'TM'), ('TM',)}"
      ]
     },
     "execution_count": 232,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ment_set.keys() | gold_ment_set.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 233,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('T', 'TM')\n",
      "6 mentions, 6 found, 3 correct.\n",
      "Precision: 0.5\n",
      "Recall:    0.5\n",
      "F1:        0.5\n",
      "FP ex.: ['שכונת קטמון', 'ארגון נפגעי ה משכנתאות', 'כביש באב אל - ואד']\n",
      "FN ex.: ['נאום אימפריית ה רשע', 'ארגון נפגעי ה משכנתאות ו חסרי ה דיור', 'ה קרב על מנזר סן סימון']\n",
      "('TM',)\n",
      "204 mentions, 178 found, 151 correct.\n",
      "Precision: 0.85\n",
      "Recall:    0.74\n",
      "F1:        0.79\n",
      "FP ex.: ['ל אקספרס', 'קטמון', 'איסט סייד', 'ירסקו', 'אולין']\n",
      "FN ex.: ['חברה קדישא', 'נווה - מונסון', 'מפם', 'טום דאיין', 'ה שדולה ה פרו - ישראלית']\n",
      "()\n",
      "244 mentions, 236 found, 194 correct.\n",
      "Precision: 0.82\n",
      "Recall:    0.8\n",
      "F1:        0.81\n",
      "FP ex.: ['גרוסבורד', 'מסצוסטס', 'שלום ה רב כהנא', 'קרן גון', 'ה שטחים']\n",
      "FN ex.: ['ה ליגה ל זכויות ה אדם', 'מסצוסטס', 'פיו', 'ה התאחדות ל עולם טוב יותר', 'פיו']\n",
      "('T',)\n",
      "45 mentions, 26 found, 23 correct.\n",
      "Precision: 0.88\n",
      "Recall:    0.51\n",
      "F1:        0.65\n",
      "FP ex.: ['ישיבת ה רעיון', 'ה ארה\"ב', 'ה מסצוסטס']\n",
      "FN ex.: ['ארה\"ב', 'תנועת ה מושבים', 'ה מצביע ה אמריקאי', 'ה קרב על סן סימון', 'תנועת ה מושבים']\n"
     ]
    }
   ],
   "source": [
    "for k in ment_set.keys() | gold_ment_set.keys():\n",
    "    print(k)\n",
    "    nem.evaluate_mentions(gold_ment_set[k], ment_set[k])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Morpheme count"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 705,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict_keys([2, 3, 1])"
      ]
     },
     "execution_count": 705,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "token_unk_sents = full_samp.groupby('sent_id').apply(get_sent, ['form', 'biose_layer0', 'morpheme_count'])\n",
    "token_unk_sents_fixed = get_sents_fixed(token_unk_sents)\n",
    "ments = [sent_to_mentions_dict(sent, sent_id, group_maker=max) for sent_id, sent in token_unk_sents_fixed]\n",
    "ment_set = get_ment_set(ments)\n",
    "ment_set.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 706,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict_keys([2, 3, 1])"
      ]
     },
     "execution_count": 706,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gold_dev_sents = dev_gold.groupby('sent_id').apply(get_sent, ['form', 'biose_layer0', 'morpheme_count'])\n",
    "gold_dev_sents_fixed = get_sents_fixed(gold_dev_sents)\n",
    "gold_ments = [sent_to_mentions_dict(sent, sent_id, group_maker=max) for sent_id, sent in gold_dev_sents_fixed]\n",
    "gold_ment_set = get_ment_set(gold_ments)\n",
    "gold_ment_set.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 707,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{1, 2, 3}"
      ]
     },
     "execution_count": 707,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ment_set.keys() | gold_ment_set.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 708,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1\n",
      "275 mentions, 260 found, 225 correct.\n",
      "Precision: 0.87\n",
      "Recall:    0.82\n",
      "F1:        0.84\n",
      "FP ex.: ['גרוסבורד', 'מסצוסטס', 'קרן גון', 'סן סימון', 'קטמון', 'יונה', 'ירסקו', 'אולין', 'קטמון', 'וייטנאם', 'רעיה']\n",
      "FN ex.: ['מפם', 'טום דאיין', 'פיו', 'מסצוסטס', 'טובייה', 'פיו', 'ורמונט', 'דה מוין', 'מקארתור', 'סמית - ריצרדסון', 'קטמון']\n",
      "2\n",
      "188 mentions, 164 found, 133 correct.\n",
      "Precision: 0.81\n",
      "Recall:    0.71\n",
      "F1:        0.76\n",
      "FP ex.: ['ליון דן', 'שלום ה רב כהנא', 'ה שמרנים', 'לוס אוחוס', 'שיקאגו', 'ה ארץ', 'מדינות ה ברית', 'ישיבת ה רעיון ה יהודי', 'מקורות', 'ה ארץ', 'קטיף ה הדרים']\n",
      "FN ex.: ['ה ליגה ל זכויות ה אדם', 'חברה קדישא', 'ה שדולה ה פרו - ישראלית', 'קרן פורד', 'ליון', 'תנועת ה מושבים', 'ה מצביע ה אמריקאי', 'מלאך ה מוות', 'אושוויץ', 'ה ברוקרים של ה רעיונות : צוותות חשיבה ו עלייתה של עלית מדיניות חדשה', 'ה קרב על סן סימון']\n",
      "3\n",
      "36 mentions, 30 found, 25 correct.\n",
      "Precision: 0.83\n",
      "Recall:    0.69\n",
      "F1:        0.76\n",
      "FP ex.: ['ה שטחים', 'ה קטיף', 'ה קטיף', 'ה קטיף', 'ה קטיף']\n",
      "FN ex.: ['ה בית ה לבן', 'ה מלחמה ה קרה', 'ה קרב על ירושלים', 'ה קרב על מנזר סן סימון', 'ה קונסוליה ה ישראלית ב שיקאגו', 'מרכז ה מידע ל זכויות ה אדם ב ה שטחים', 'ה בית ה לבן', 'איסט סייד ה תחתית', 'ה עולם ה שלישי', 'ה מלחמה ה קרה', 'ה עבודה ו ה רווחה']\n"
     ]
    }
   ],
   "source": [
    "for k in ment_set.keys() | gold_ment_set.keys():\n",
    "    print(k)\n",
    "    nem.evaluate_mentions(gold_ment_set[k], ment_set[k], examples=11)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "['ה בית ה לבן', 'ה מלחמה ה קרה', 'ה קרב על ירושלים', 'ה קרב על מנזר סן סימון', 'ה קונסוליה ה ישראלית ב שיקאגו', 'מרכז ה מידע ל זכויות ה אדם ב ה שטחים', 'ה בית ה לבן', 'איסט סייד ה תחתית', 'ה עולם ה שלישי', 'ה מלחמה ה קרה']\n",
    "['ה בית ה לבן', 'ה מלחמה ה קרה', 'ה קרב על ירושלים', 'ה קרב על מנזר סן סימון', 'ה קונסוליה ה ישראלית ב שיקאגו', 'מרכז ה מידע ל זכויות ה אדם ב ה שטחים', 'ה בית ה לבן', 'איסט סייד ה תחתית', 'ה עולם ה שלישי', 'ה מלחמה ה קרה', 'ה עבודה ו ה רווחה']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 683,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    [(עשרות, O), (אנשים, O), (מגיעים, O), (מ, O), ...\n",
       "1    [(תופעה, O), (זו, O), (התבררה, O), (אתמול, O),...\n",
       "2    [(יו\"ר, O), (ה, O), (וועדה, O), (,, O), (ח\"כ, ...\n",
       "3    [(מ, O), (צד, O), (אחד, O), (רוצה, O), (ה, B-O...\n",
       "4    [(נמיר, S-PER), (הודיעה, O), (כי, O), (תפנה, O...\n",
       "dtype: object"
      ]
     },
     "execution_count": 683,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pred_samp = nem.read_file_sents(os.path.join('final_setup/decode_output/morph_dev_gold.morph.char_cnn.glv_yap.44_seed.bmes'))\n",
    "pred_samp.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 684,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sent_id</th>\n",
       "      <th>id</th>\n",
       "      <th>biose_layer0</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>S-GPE</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   sent_id  id biose_layer0\n",
       "0        1   1            O\n",
       "1        1   2            O\n",
       "2        1   3            O\n",
       "3        1   4            O\n",
       "4        1   5        S-GPE"
      ]
     },
     "execution_count": 684,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "for_df = []\n",
    "for i, line in pred_samp.iteritems():\n",
    "    sent_id=i+1\n",
    "    for j, (tok, bio) in enumerate(line):\n",
    "        for_df.append((sent_id, j+1, bio))\n",
    "ner_samp = pd.DataFrame(for_df, columns=['sent_id', 'id', 'biose_layer0'])\n",
    "ner_samp.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 685,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>form</th>\n",
       "      <th>lemma</th>\n",
       "      <th>upostag</th>\n",
       "      <th>xpostag</th>\n",
       "      <th>feats</th>\n",
       "      <th>head</th>\n",
       "      <th>deprel</th>\n",
       "      <th>deps</th>\n",
       "      <th>misc</th>\n",
       "      <th>sent_id</th>\n",
       "      <th>token_id</th>\n",
       "      <th>token_str</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>עשרות</td>\n",
       "      <td>עשר</td>\n",
       "      <td>CDT</td>\n",
       "      <td>CDT</td>\n",
       "      <td>gen=F|num=P</td>\n",
       "      <td>2</td>\n",
       "      <td>num</td>\n",
       "      <td>_</td>\n",
       "      <td>_</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>עשרות</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>אנשים</td>\n",
       "      <td>איש</td>\n",
       "      <td>NN</td>\n",
       "      <td>NN</td>\n",
       "      <td>gen=M|num=P</td>\n",
       "      <td>3</td>\n",
       "      <td>subj</td>\n",
       "      <td>_</td>\n",
       "      <td>_</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>אנשים</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>מגיעים</td>\n",
       "      <td>הגיע</td>\n",
       "      <td>BN</td>\n",
       "      <td>BN</td>\n",
       "      <td>gen=M|num=P|per=A</td>\n",
       "      <td>14</td>\n",
       "      <td>conj</td>\n",
       "      <td>_</td>\n",
       "      <td>_</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>מגיעים</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>מ</td>\n",
       "      <td>מ</td>\n",
       "      <td>PREPOSITION</td>\n",
       "      <td>PREPOSITION</td>\n",
       "      <td>NaN</td>\n",
       "      <td>3</td>\n",
       "      <td>comp</td>\n",
       "      <td>_</td>\n",
       "      <td>_</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>מתאילנד</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>תאילנד</td>\n",
       "      <td>תאילנד</td>\n",
       "      <td>NNP</td>\n",
       "      <td>NNP</td>\n",
       "      <td>gen=F|num=S</td>\n",
       "      <td>4</td>\n",
       "      <td>pobj</td>\n",
       "      <td>_</td>\n",
       "      <td>_</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>מתאילנד</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   id    form   lemma      upostag      xpostag              feats  head  \\\n",
       "0   1   עשרות     עשר          CDT          CDT        gen=F|num=P     2   \n",
       "1   2   אנשים     איש           NN           NN        gen=M|num=P     3   \n",
       "2   3  מגיעים    הגיע           BN           BN  gen=M|num=P|per=A    14   \n",
       "3   4       מ       מ  PREPOSITION  PREPOSITION                NaN     3   \n",
       "4   5  תאילנד  תאילנד          NNP          NNP        gen=F|num=S     4   \n",
       "\n",
       "  deprel deps misc  sent_id token_id token_str  \n",
       "0    num    _    _        1        1     עשרות  \n",
       "1   subj    _    _        1        2     אנשים  \n",
       "2   conj    _    _        1        3    מגיעים  \n",
       "3   comp    _    _        1        4   מתאילנד  \n",
       "4   pobj    _    _        1        4   מתאילנד  "
      ]
     },
     "execution_count": 685,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "yap_out = bclm.read_yap_output(treebank_set=None,\n",
    "                               tokens_path=bclm.TREEBANK_TOKEN_PATHS['dev'], \n",
    "                                dep_path=txt_map[('dev', 'char_cnn', 'glv_yap', 'conll')],\n",
    "                                map_path=txt_map[('dev', 'char_cnn', 'glv_yap', 'map')],)\n",
    "yap_out.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 686,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>form</th>\n",
       "      <th>lemma</th>\n",
       "      <th>upostag</th>\n",
       "      <th>xpostag</th>\n",
       "      <th>feats</th>\n",
       "      <th>head</th>\n",
       "      <th>deprel</th>\n",
       "      <th>deps</th>\n",
       "      <th>misc</th>\n",
       "      <th>sent_id</th>\n",
       "      <th>token_id</th>\n",
       "      <th>token_str</th>\n",
       "      <th>biose_layer0</th>\n",
       "      <th>token_unk</th>\n",
       "      <th>morph_unk</th>\n",
       "      <th>lemma_unk</th>\n",
       "      <th>unk_type</th>\n",
       "      <th>has_ner</th>\n",
       "      <th>morpheme_count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>11188</th>\n",
       "      <td>1</td>\n",
       "      <td>מתחיל</td>\n",
       "      <td>התחייל</td>\n",
       "      <td>BNT</td>\n",
       "      <td>BNT</td>\n",
       "      <td>gen=M|num=S|per=A</td>\n",
       "      <td>0</td>\n",
       "      <td>ROOT</td>\n",
       "      <td>_</td>\n",
       "      <td>_</td>\n",
       "      <td>500</td>\n",
       "      <td>1</td>\n",
       "      <td>מתחיל</td>\n",
       "      <td>O</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td></td>\n",
       "      <td>False</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11189</th>\n",
       "      <td>2</td>\n",
       "      <td>מסע</td>\n",
       "      <td>מסע</td>\n",
       "      <td>NNT</td>\n",
       "      <td>NNT</td>\n",
       "      <td>gen=M|num=S</td>\n",
       "      <td>1</td>\n",
       "      <td>gobj</td>\n",
       "      <td>_</td>\n",
       "      <td>_</td>\n",
       "      <td>500</td>\n",
       "      <td>2</td>\n",
       "      <td>מסע</td>\n",
       "      <td>O</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td></td>\n",
       "      <td>False</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11190</th>\n",
       "      <td>3</td>\n",
       "      <td>ה</td>\n",
       "      <td>ה</td>\n",
       "      <td>DEF</td>\n",
       "      <td>DEF</td>\n",
       "      <td>NaN</td>\n",
       "      <td>4</td>\n",
       "      <td>def</td>\n",
       "      <td>_</td>\n",
       "      <td>_</td>\n",
       "      <td>500</td>\n",
       "      <td>3</td>\n",
       "      <td>הנקמה</td>\n",
       "      <td>O</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td></td>\n",
       "      <td>False</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11191</th>\n",
       "      <td>4</td>\n",
       "      <td>נקמה</td>\n",
       "      <td>נקמה</td>\n",
       "      <td>NN</td>\n",
       "      <td>NN</td>\n",
       "      <td>gen=F|num=S</td>\n",
       "      <td>2</td>\n",
       "      <td>gobj</td>\n",
       "      <td>_</td>\n",
       "      <td>_</td>\n",
       "      <td>500</td>\n",
       "      <td>3</td>\n",
       "      <td>הנקמה</td>\n",
       "      <td>O</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td></td>\n",
       "      <td>False</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11192</th>\n",
       "      <td>5</td>\n",
       "      <td>.</td>\n",
       "      <td>NaN</td>\n",
       "      <td>yyDOT</td>\n",
       "      <td>yyDOT</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>punct</td>\n",
       "      <td>_</td>\n",
       "      <td>_</td>\n",
       "      <td>500</td>\n",
       "      <td>4</td>\n",
       "      <td>.</td>\n",
       "      <td>O</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td></td>\n",
       "      <td>False</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       id   form   lemma upostag xpostag              feats  head deprel deps  \\\n",
       "11188   1  מתחיל  התחייל     BNT     BNT  gen=M|num=S|per=A     0   ROOT    _   \n",
       "11189   2    מסע     מסע     NNT     NNT        gen=M|num=S     1   gobj    _   \n",
       "11190   3      ה       ה     DEF     DEF                NaN     4    def    _   \n",
       "11191   4   נקמה    נקמה      NN      NN        gen=F|num=S     2   gobj    _   \n",
       "11192   5      .     NaN   yyDOT   yyDOT                NaN     1  punct    _   \n",
       "\n",
       "      misc  sent_id token_id token_str biose_layer0  token_unk  morph_unk  \\\n",
       "11188    _      500        1     מתחיל            O      False      False   \n",
       "11189    _      500        2       מסע            O      False      False   \n",
       "11190    _      500        3     הנקמה            O      False      False   \n",
       "11191    _      500        3     הנקמה            O      False      False   \n",
       "11192    _      500        4         .            O      False      False   \n",
       "\n",
       "       lemma_unk unk_type  has_ner  morpheme_count  \n",
       "11188      False             False               1  \n",
       "11189      False             False               1  \n",
       "11190      False             False               2  \n",
       "11191      False             False               2  \n",
       "11192      False             False               1  "
      ]
     },
     "execution_count": 686,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "full_samp = yap_out.merge(ner_samp)\n",
    "full_samp = full_samp.merge(dev_unk, how='left')\n",
    "full_samp.tail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 687,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>form</th>\n",
       "      <th>lemma</th>\n",
       "      <th>upostag</th>\n",
       "      <th>xpostag</th>\n",
       "      <th>feats</th>\n",
       "      <th>token_id</th>\n",
       "      <th>sent_id</th>\n",
       "      <th>token_str</th>\n",
       "      <th>global_sent_id</th>\n",
       "      <th>...</th>\n",
       "      <th>biose_layer0</th>\n",
       "      <th>biose_layer1</th>\n",
       "      <th>biose_layer2</th>\n",
       "      <th>biose_layer3</th>\n",
       "      <th>token_unk</th>\n",
       "      <th>morph_unk</th>\n",
       "      <th>lemma_unk</th>\n",
       "      <th>unk_type</th>\n",
       "      <th>has_ner</th>\n",
       "      <th>morpheme_count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>11296</th>\n",
       "      <td>1</td>\n",
       "      <td>מתחיל</td>\n",
       "      <td>התחיל</td>\n",
       "      <td>BN</td>\n",
       "      <td>BN</td>\n",
       "      <td>gen=M|num=S|per=A</td>\n",
       "      <td>1</td>\n",
       "      <td>500</td>\n",
       "      <td>מתחיל</td>\n",
       "      <td>500</td>\n",
       "      <td>...</td>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td></td>\n",
       "      <td>False</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11297</th>\n",
       "      <td>2</td>\n",
       "      <td>מסע</td>\n",
       "      <td>מסע</td>\n",
       "      <td>NNT</td>\n",
       "      <td>NNT</td>\n",
       "      <td>gen=M|num=S</td>\n",
       "      <td>2</td>\n",
       "      <td>500</td>\n",
       "      <td>מסע</td>\n",
       "      <td>500</td>\n",
       "      <td>...</td>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td></td>\n",
       "      <td>False</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11298</th>\n",
       "      <td>3</td>\n",
       "      <td>ה</td>\n",
       "      <td>ה</td>\n",
       "      <td>DEF</td>\n",
       "      <td>DEF</td>\n",
       "      <td>_</td>\n",
       "      <td>3</td>\n",
       "      <td>500</td>\n",
       "      <td>הנקמה</td>\n",
       "      <td>500</td>\n",
       "      <td>...</td>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td></td>\n",
       "      <td>False</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11299</th>\n",
       "      <td>4</td>\n",
       "      <td>נקמה</td>\n",
       "      <td>נקמה</td>\n",
       "      <td>NN</td>\n",
       "      <td>NN</td>\n",
       "      <td>gen=F|num=S</td>\n",
       "      <td>3</td>\n",
       "      <td>500</td>\n",
       "      <td>הנקמה</td>\n",
       "      <td>500</td>\n",
       "      <td>...</td>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td></td>\n",
       "      <td>False</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11300</th>\n",
       "      <td>5</td>\n",
       "      <td>.</td>\n",
       "      <td>_</td>\n",
       "      <td>yyDOT</td>\n",
       "      <td>yyDOT</td>\n",
       "      <td>_</td>\n",
       "      <td>4</td>\n",
       "      <td>500</td>\n",
       "      <td>.</td>\n",
       "      <td>500</td>\n",
       "      <td>...</td>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td></td>\n",
       "      <td>False</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 33 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       id   form  lemma upostag xpostag              feats  token_id  sent_id  \\\n",
       "11296   1  מתחיל  התחיל      BN      BN  gen=M|num=S|per=A         1      500   \n",
       "11297   2    מסע    מסע     NNT     NNT        gen=M|num=S         2      500   \n",
       "11298   3      ה      ה     DEF     DEF                  _         3      500   \n",
       "11299   4   נקמה   נקמה      NN      NN        gen=F|num=S         3      500   \n",
       "11300   5      .      _   yyDOT   yyDOT                  _         4      500   \n",
       "\n",
       "      token_str  global_sent_id  ...  biose_layer0 biose_layer1 biose_layer2  \\\n",
       "11296     מתחיל             500  ...             O            O            O   \n",
       "11297       מסע             500  ...             O            O            O   \n",
       "11298     הנקמה             500  ...             O            O            O   \n",
       "11299     הנקמה             500  ...             O            O            O   \n",
       "11300         .             500  ...             O            O            O   \n",
       "\n",
       "       biose_layer3 token_unk  morph_unk lemma_unk unk_type has_ner  \\\n",
       "11296             O     False      False     False            False   \n",
       "11297             O     False      False     False            False   \n",
       "11298             O     False      False     False            False   \n",
       "11299             O     False      False     False            False   \n",
       "11300             O     False      False     False            False   \n",
       "\n",
       "      morpheme_count  \n",
       "11296              1  \n",
       "11297              1  \n",
       "11298              2  \n",
       "11299              2  \n",
       "11300              1  \n",
       "\n",
       "[5 rows x 33 columns]"
      ]
     },
     "execution_count": 687,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "full_samp = dev_gold.copy()\n",
    "full_samp['biose_layer0'] = ner_samp['biose_layer0']\n",
    "full_samp = full_samp.merge(dev_unk, how='left')\n",
    "full_samp.tail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 688,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict_keys([2, 3, 1])"
      ]
     },
     "execution_count": 688,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "token_unk_sents = full_samp.groupby('sent_id').apply(get_sent, ['form', 'biose_layer0', 'morpheme_count'])\n",
    "token_unk_sents_fixed = get_sents_fixed(token_unk_sents)\n",
    "ments = [sent_to_mentions_dict(sent, sent_id, group_maker=max) for sent_id, sent in token_unk_sents_fixed]\n",
    "ment_set = get_ment_set(ments)\n",
    "ment_set.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 689,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict_keys([2, 3, 1])"
      ]
     },
     "execution_count": 689,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gold_dev_sents = dev_gold.groupby('sent_id').apply(get_sent, ['form', 'biose_layer0', 'morpheme_count'])\n",
    "gold_dev_sents_fixed = get_sents_fixed(gold_dev_sents)\n",
    "gold_ments = [sent_to_mentions_dict(sent, sent_id, group_maker=max) for sent_id, sent in gold_dev_sents_fixed]\n",
    "gold_ment_set = get_ment_set(gold_ments)\n",
    "gold_ment_set.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 690,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{1, 2, 3}"
      ]
     },
     "execution_count": 690,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ment_set.keys() | gold_ment_set.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 691,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1\n",
      "275 mentions, 268 found, 220 correct.\n",
      "Precision: 0.82\n",
      "Recall:    0.8\n",
      "F1:        0.81\n",
      "FP ex.: ['אוחדה', 'גרוסבורד', 'הדסון', 'רץ', 'קדישא', 'יונה', 'אולין', 'ברוקינגס', 'רצ\"ב', 'וייטנאם']\n",
      "FN ex.: ['קול ישראל', 'מפם', 'טום דאיין', 'פיו', 'מסצוסטס', 'טובייה', 'פיו', 'ריצארדס', 'דה מוין', 'מקארתור']\n",
      "2\n",
      "188 mentions, 156 found, 126 correct.\n",
      "Precision: 0.81\n",
      "Recall:    0.67\n",
      "F1:        0.73\n",
      "FP ex.: ['ליון דן', 'ה משטרה', 'שיקאגו', 'ה ארץ', 'מדינות ה ברית', 'מקורות', 'ה ארץ', 'שמואל ה נביא', 'עלי', 'מושלת טקסס']\n",
      "FN ex.: ['ה ליגה ל זכויות ה אדם', 'חברה קדישא', 'נווה - מונסון', 'ה שדולה ה פרו - ישראלית', 'קרן פורד', 'ליון', 'ה התאחדות ל עולם טוב יותר', 'ה מצביע ה אמריקאי', 'מלאך ה מוות', 'ה מיליציה ה צרפתית']\n",
      "3\n",
      "36 mentions, 28 found, 26 correct.\n",
      "Precision: 0.93\n",
      "Recall:    0.72\n",
      "F1:        0.81\n",
      "FP ex.: ['ה שטחים', 'ה איסט סייד']\n",
      "FN ex.: ['ה בית ה לבן', 'ה מלחמה ה קרה', 'ה קרב על ירושלים', 'ה קרב על מנזר סן סימון', 'מרכז ה מידע ל זכויות ה אדם ב ה שטחים', 'ה בית ה לבן', 'איסט סייד ה תחתית', 'ה עולם ה שלישי', 'ה מלחמה ה קרה', 'ה קונסוליה ה ישראלית ב שיקאגו']\n"
     ]
    }
   ],
   "source": [
    "for k in ment_set.keys() | gold_ment_set.keys():\n",
    "    print(k)\n",
    "    nem.evaluate_mentions(gold_ment_set[k], ment_set[k], examples=10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Check morpheme count together with unk_type"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 238,
   "metadata": {},
   "outputs": [],
   "source": [
    "def gm(l):\n",
    "    uts, mcs = list(zip(*l))\n",
    "    ut = ut_gm(uts)\n",
    "    mc = max(mcs)\n",
    "    return ut, mc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 239,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict_keys([(('T',), 2), ((), 2), ((), 3), ((), 1), (('T',), 1), (('TM',), 1), (('T', 'TM'), 1), (('TM',), 2), (('T', 'TM'), 2), (('TM',), 3), (('T',), 3)])"
      ]
     },
     "execution_count": 239,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "token_unk_sents = full_samp.groupby('sent_id').apply(get_sent, ['form', 'biose_layer0', 'unk_type', 'morpheme_count'])\n",
    "token_unk_sents_fixed = get_sents_fixed(token_unk_sents)\n",
    "ments = [sent_to_mentions_dict(sent, sent_id, group_maker=gm) for sent_id, sent in token_unk_sents_fixed]\n",
    "ment_set = get_ment_set(ments)\n",
    "ment_set.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 240,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict_keys([(('T',), 2), ((), 2), ((), 3), ((), 1), (('T',), 3), (('T',), 1), (('TM',), 1), (('T', 'TM'), 3), (('TM',), 2), (('T', 'TM'), 2), (('TM',), 3)])"
      ]
     },
     "execution_count": 240,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gold_dev_sents = dev_gold.groupby('sent_id').apply(get_sent, ['form', 'biose_layer0', 'unk_type', 'morpheme_count'])\n",
    "gold_dev_sents_fixed = get_sents_fixed(gold_dev_sents)\n",
    "gold_ments = [sent_to_mentions_dict(sent, sent_id, group_maker=gm) for sent_id, sent in gold_dev_sents_fixed]\n",
    "gold_ment_set = get_ment_set(gold_ments)\n",
    "gold_ment_set.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 241,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{((), 1),\n",
       " ((), 2),\n",
       " ((), 3),\n",
       " (('T',), 1),\n",
       " (('T',), 2),\n",
       " (('T',), 3),\n",
       " (('T', 'TM'), 1),\n",
       " (('T', 'TM'), 2),\n",
       " (('T', 'TM'), 3),\n",
       " (('TM',), 1),\n",
       " (('TM',), 2),\n",
       " (('TM',), 3)}"
      ]
     },
     "execution_count": 241,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ment_set.keys() | gold_ment_set.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 242,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(('TM',), 3)\n",
      "2 mentions, 5 found, 0 correct.\n",
      "Precision: 0.0\n",
      "Recall:    0.0\n",
      "F1:        -1\n",
      "FP ex.: ['ה קטיף', 'איסט סייד', 'ה קטיף', 'ה קטיף', 'ה קטיף']\n",
      "FN ex.: ['איסט סייד ה תחתית', 'ה קונסוליה ה ישראלית ב שיקאגו']\n",
      "(('TM',), 2)\n",
      "47 mentions, 30 found, 26 correct.\n",
      "Precision: 0.87\n",
      "Recall:    0.55\n",
      "F1:        0.68\n",
      "FP ex.: ['נווה - מונסון', 'הרי בראדלו', 'קרן פורד שמרנים', 'קטיף ה הדרים']\n",
      "FN ex.: ['חברה קדישא', 'נווה - מונסון', 'ה שדולה ה פרו - ישראלית', 'ליון', 'קרן פורד']\n",
      "(('T', 'TM'), 3)\n",
      "1 mentions, 0 found, 0 correct.\n",
      "Precision: -1\n",
      "Recall:    0.0\n",
      "F1:        0.0\n",
      "FP ex.: []\n",
      "FN ex.: ['ה קרב על מנזר סן סימון']\n",
      "(('T', 'TM'), 2)\n",
      "5 mentions, 4 found, 3 correct.\n",
      "Precision: 0.75\n",
      "Recall:    0.6\n",
      "F1:        0.67\n",
      "FP ex.: ['ארגון נפגעי ה משכנתאות']\n",
      "FN ex.: ['נאום אימפריית ה רשע', 'ארגון נפגעי ה משכנתאות ו חסרי ה דיור']\n",
      "(('T',), 2)\n",
      "34 mentions, 20 found, 19 correct.\n",
      "Precision: 0.95\n",
      "Recall:    0.56\n",
      "F1:        0.7\n",
      "FP ex.: ['ישיבת ה רעיון']\n",
      "FN ex.: ['ה קרב על סן סימון', 'תנועת ה מושבים', 'ה בית ה לבן', 'ה בית ה לבן', 'ישיבת ה רעיון ה יהודי']\n",
      "(('T', 'TM'), 1)\n",
      "0 mentions, 2 found, 0 correct.\n",
      "Precision: 0.0\n",
      "Recall:    -1\n",
      "F1:        0.0\n",
      "FP ex.: ['שכונת קטמון', 'כביש באב אל - ואד']\n",
      "FN ex.: []\n",
      "(('T',), 3)\n",
      "9 mentions, 4 found, 2 correct.\n",
      "Precision: 0.5\n",
      "Recall:    0.22\n",
      "F1:        0.31\n",
      "FP ex.: ['ה ארה\"ב', 'ה מסצוסטס']\n",
      "FN ex.: ['ה בית ה לבן', 'ארה\"ב', 'ה מלחמה ה קרה', 'מסצוסטס', 'ה בית ה לבן']\n",
      "((), 2)\n",
      "102 mentions, 102 found, 80 correct.\n",
      "Precision: 0.78\n",
      "Recall:    0.78\n",
      "F1:        0.78\n",
      "FP ex.: ['שלום ה רב כהנא', 'ה שמרנים', 'שיקאגו', 'ה ארץ', 'מדינות ה ברית']\n",
      "FN ex.: ['ה ליגה ל זכויות ה אדם', 'ה התאחדות ל עולם טוב יותר', 'אושוויץ', 'ה ארץ', 'ה ארץ']\n",
      "((), 3)\n",
      "24 mentions, 20 found, 18 correct.\n",
      "Precision: 0.9\n",
      "Recall:    0.75\n",
      "F1:        0.82\n",
      "FP ex.: ['ה שטחים', 'ה ארץ']\n",
      "FN ex.: ['ה ארץ', 'ה ארץ', 'ה קרב על ירושלים', 'מרכז ה מידע ל זכויות ה אדם ב ה שטחים', 'ה כנסת']\n",
      "(('T',), 1)\n",
      "2 mentions, 2 found, 2 correct.\n",
      "Precision: 1.0\n",
      "Recall:    1.0\n",
      "F1:        1.0\n",
      "FP ex.: []\n",
      "FN ex.: []\n",
      "((), 1)\n",
      "118 mentions, 114 found, 96 correct.\n",
      "Precision: 0.84\n",
      "Recall:    0.81\n",
      "F1:        0.83\n",
      "FP ex.: ['מסצוסטס', 'הראל', 'גרוסבורד', 'מסצוסטס', 'מחוז פאריס']\n",
      "FN ex.: ['מסצוסטס', 'פיו', 'פיו', 'ה', 'מחוז פאריס']\n",
      "(('TM',), 1)\n",
      "155 mentions, 143 found, 125 correct.\n",
      "Precision: 0.87\n",
      "Recall:    0.81\n",
      "F1:        0.84\n",
      "FP ex.: ['מנזר סן סימון', 'פראפראז', 'מכון ה דסון', 'סמית - ריצרדסון', 'ל אקספרס']\n",
      "FN ex.: ['מפם', 'טום דאיין', 'טובייה', 'ורמונט', 'לאקספרס']\n"
     ]
    }
   ],
   "source": [
    "for k in ment_set.keys() | gold_ment_set.keys():\n",
    "    print(k)\n",
    "    nem.evaluate_mentions(gold_ment_set[k], ment_set[k])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Check length of mention"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 243,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict_keys([1, 6, 2, 3, 4, 5, 7])"
      ]
     },
     "execution_count": 243,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "token_unk_sents = full_samp.groupby('sent_id').apply(get_sent, ['form', 'biose_layer0', 'unk_type', 'morpheme_count'])\n",
    "token_unk_sents_fixed = get_sents_fixed(token_unk_sents)\n",
    "ments = [sent_to_mentions_dict(sent, sent_id, group_maker=lambda x: None, ment_len=True) for sent_id, sent in token_unk_sents_fixed]\n",
    "ment_set = get_ment_set(ments)\n",
    "ment_set.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 244,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict_keys([1, 6, 2, 5, 3, 4, 8, 7, 10, 14])"
      ]
     },
     "execution_count": 244,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gold_dev_sents = dev_gold.groupby('sent_id').apply(get_sent, ['form', 'biose_layer0', 'unk_type', 'morpheme_count'])\n",
    "gold_dev_sents_fixed = get_sents_fixed(gold_dev_sents)\n",
    "gold_ments = [sent_to_mentions_dict(sent, sent_id, group_maker=lambda x: None, ment_len=True) for sent_id, sent in gold_dev_sents_fixed]\n",
    "gold_ment_set = get_ment_set(gold_ments)\n",
    "gold_ment_set.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 245,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{1, 2, 3, 4, 5, 6, 7, 8, 10, 14}"
      ]
     },
     "execution_count": 245,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ment_set.keys() | gold_ment_set.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 246,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1\n",
      "228 mentions, 207 found, 183 correct.\n",
      "Precision: 0.88\n",
      "Recall:    0.8\n",
      "F1:        0.84\n",
      "FP ex.: ['גרוסבורד', 'מסצוסטס', 'מסצוסטס', 'קטמון', 'יונה']\n",
      "FN ex.: ['ארה\"ב', 'מפם', 'פיו', 'מסצוסטס', 'ליון']\n",
      "2\n",
      "154 mentions, 152 found, 123 correct.\n",
      "Precision: 0.81\n",
      "Recall:    0.8\n",
      "F1:        0.8\n",
      "FP ex.: ['קרן גון', 'ה שמרנים', 'ל אקספרס', 'ה שטחים', 'איסט סייד']\n",
      "FN ex.: ['חברה קדישא', 'טום דאיין', 'קרן פורד', 'ה ארץ', 'ה כנסת']\n",
      "3\n",
      "48 mentions, 44 found, 30 correct.\n",
      "Precision: 0.68\n",
      "Recall:    0.62\n",
      "F1:        0.65\n",
      "FP ex.: ['מנזר סן סימון', 'מכון ה דסון', 'סמית - ריצרדסון', 'נווה - מונסון', 'ישיבת ה רעיון']\n",
      "FN ex.: ['מלחמת ה עצמאות', 'ירושלים ה מערבית', 'נווה - מונסון', 'תנועת ה מושבים', 'אמריקן אקונומיק ריוויו']\n",
      "4\n",
      "45 mentions, 29 found, 24 correct.\n",
      "Precision: 0.83\n",
      "Recall:    0.53\n",
      "F1:        0.65\n",
      "FP ex.: ['שלום ה רב כהנא', 'ה מלחמת ה עצמאות', 'הר - ה בית', 'ארגון נפגעי ה משכנתאות', 'שכונת שמואל ה נביא']\n",
      "FN ex.: ['ה מצביע ה אמריקאי', 'ה עולם ה שלישי', 'לינד ו הרי בראדלו', 'ה קרב על ירושלים', 'איסט סייד ה תחתית']\n",
      "5\n",
      "12 mentions, 10 found, 7 correct.\n",
      "Precision: 0.7\n",
      "Recall:    0.58\n",
      "F1:        0.64\n",
      "FP ex.: ['ישיבת ה רעיון ה יהודי', 'אייפא\"ק טום דאיין את הארקין', 'כביש באב אל - ואד']\n",
      "FN ex.: ['ה קרב על סן סימון', 'ישיבת ה רעיון ה יהודי', 'ישיבת ה רעיון ה יהודי', 'ה קרב על סן סימון', 'ה עבודה ו ה רווחה']\n",
      "6\n",
      "8 mentions, 3 found, 3 correct.\n",
      "Precision: 1.0\n",
      "Recall:    0.38\n",
      "F1:        0.55\n",
      "FP ex.: []\n",
      "FN ex.: ['ה ליגה ל זכויות ה אדם', 'ה שדולה ה פרו - ישראלית', 'ה התאחדות ל עולם טוב יותר', 'ה קרב על מנזר סן סימון', 'ה קונסוליה ה ישראלית ב שיקאגו']\n",
      "7\n",
      "1 mentions, 1 found, 1 correct.\n",
      "Precision: 1.0\n",
      "Recall:    1.0\n",
      "F1:        1.0\n",
      "FP ex.: []\n",
      "FN ex.: []\n",
      "8\n",
      "1 mentions, 0 found, 0 correct.\n",
      "Precision: -1\n",
      "Recall:    0.0\n",
      "F1:        0.0\n",
      "FP ex.: []\n",
      "FN ex.: ['ארגון נפגעי ה משכנתאות ו חסרי ה דיור']\n",
      "10\n",
      "1 mentions, 0 found, 0 correct.\n",
      "Precision: -1\n",
      "Recall:    0.0\n",
      "F1:        0.0\n",
      "FP ex.: []\n",
      "FN ex.: ['מרכז ה מידע ל זכויות ה אדם ב ה שטחים']\n",
      "14\n",
      "1 mentions, 0 found, 0 correct.\n",
      "Precision: -1\n",
      "Recall:    0.0\n",
      "F1:        0.0\n",
      "FP ex.: []\n",
      "FN ex.: ['ה ברוקרים של ה רעיונות : צוותות חשיבה ו עלייתה של עלית מדיניות חדשה']\n"
     ]
    }
   ],
   "source": [
    "for k in sorted(ment_set.keys() | gold_ment_set.keys()):\n",
    "    print(k)\n",
    "    nem.evaluate_mentions(gold_ment_set[k], ment_set[k])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Length+unk_type"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 247,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict_keys([(('T',), 1), ((), 1), ((), 6), ((), 2), (('T',), 2), (('T',), 3), ((), 3), ((), 4), ((), 5), (('TM',), 2), (('T', 'TM'), 2), (('TM',), 1), (('TM',), 3), (('T', 'TM'), 5), (('T', 'TM'), 4), ((), 7), (('TM',), 5), (('T',), 5), (('TM',), 4)])"
      ]
     },
     "execution_count": 247,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "token_unk_sents = full_samp.groupby('sent_id').apply(get_sent, ['form', 'biose_layer0', 'unk_type'])\n",
    "token_unk_sents_fixed = get_sents_fixed(token_unk_sents)\n",
    "ments = [sent_to_mentions_dict(sent, sent_id, group_maker=ut_gm, ment_len=True) for sent_id, sent in token_unk_sents_fixed]\n",
    "ment_set = get_ment_set(ments)\n",
    "ment_set.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 248,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict_keys([(('T',), 1), ((), 1), ((), 6), ((), 2), (('T',), 5), (('T',), 3), (('T',), 2), ((), 3), ((), 4), ((), 5), (('TM',), 2), (('TM',), 1), (('TM',), 3), (('T', 'TM'), 6), (('TM',), 4), (('T', 'TM'), 8), ((), 7), (('T',), 4), (('TM',), 6), ((), 10), (('T', 'TM'), 2), (('T', 'TM'), 4), (('TM',), 14)])"
      ]
     },
     "execution_count": 248,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gold_dev_sents = dev_gold.groupby('sent_id').apply(get_sent, ['form', 'biose_layer0', 'unk_type'])\n",
    "gold_dev_sents_fixed = get_sents_fixed(gold_dev_sents)\n",
    "gold_ments = [sent_to_mentions_dict(sent, sent_id, group_maker=ut_gm, ment_len=True) for sent_id, sent in gold_dev_sents_fixed]\n",
    "gold_ment_set = get_ment_set(gold_ments)\n",
    "gold_ment_set.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 249,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{((), 1),\n",
       " ((), 2),\n",
       " ((), 3),\n",
       " ((), 4),\n",
       " ((), 5),\n",
       " ((), 6),\n",
       " ((), 7),\n",
       " ((), 10),\n",
       " (('T',), 1),\n",
       " (('T',), 2),\n",
       " (('T',), 3),\n",
       " (('T',), 4),\n",
       " (('T',), 5),\n",
       " (('T', 'TM'), 2),\n",
       " (('T', 'TM'), 4),\n",
       " (('T', 'TM'), 5),\n",
       " (('T', 'TM'), 6),\n",
       " (('T', 'TM'), 8),\n",
       " (('TM',), 1),\n",
       " (('TM',), 2),\n",
       " (('TM',), 3),\n",
       " (('TM',), 4),\n",
       " (('TM',), 5),\n",
       " (('TM',), 6),\n",
       " (('TM',), 14)}"
      ]
     },
     "execution_count": 249,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ment_set.keys() | gold_ment_set.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 250,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "((), 1)\n",
      "110 mentions, 108 found, 92 correct.\n",
      "Precision: 0.85\n",
      "Recall:    0.84\n",
      "F1:        0.84\n",
      "FP ex.: ['מסצוסטס', 'הראל', 'גרוסבורד', 'מסצוסטס', 'מסצוסטס']\n",
      "FN ex.: ['ה', 'כך', 'וייטנאם', 'מסצוסטס', 'פיו']\n",
      "((), 2)\n",
      "69 mentions, 69 found, 53 correct.\n",
      "Precision: 0.77\n",
      "Recall:    0.77\n",
      "F1:        0.77\n",
      "FP ex.: ['מחוז פאריס', 'ה שמרנים', 'סן סימון', 'ה שטחים', 'קרן גון']\n",
      "FN ex.: ['ה ארץ', 'מחוז פאריס', 'ה גרמנים', 'סנטר 1', 'ניו גרסי']\n",
      "((), 3)\n",
      "23 mentions, 23 found, 18 correct.\n",
      "Precision: 0.78\n",
      "Recall:    0.78\n",
      "F1:        0.78\n",
      "FP ex.: ['רצח ה נזירות', 'מדינות ה ברית', 'רחוב גבעת שאול', 'גורג ה מסכן', 'ה שירות פועל']\n",
      "FN ex.: ['בית מספר 3', 'ירושלים ה מערבית', 'רחוב גבעת שאול', 'מלחמת ה עצמאות', 'ניו - יורק']\n",
      "((), 4)\n",
      "29 mentions, 26 found, 22 correct.\n",
      "Precision: 0.85\n",
      "Recall:    0.76\n",
      "F1:        0.8\n",
      "FP ex.: ['הר - ה בית', 'שכונת שמואל ה נביא', 'שלום ה רב כהנא', 'ה מלחמת ה עצמאות']\n",
      "FN ex.: ['חטיבת \" הראל \"', 'פרס נובל ל שלום', 'ה קרב על ירושלים', 'הר - ה בית', 'שכונת שמואל ה נביא']\n",
      "((), 5)\n",
      "6 mentions, 6 found, 5 correct.\n",
      "Precision: 0.83\n",
      "Recall:    0.83\n",
      "F1:        0.83\n",
      "FP ex.: ['ישיבת ה רעיון ה יהודי']\n",
      "FN ex.: ['ישיבת ה רעיון ה יהודי']\n",
      "((), 6)\n",
      "5 mentions, 3 found, 3 correct.\n",
      "Precision: 1.0\n",
      "Recall:    0.6\n",
      "F1:        0.75\n",
      "FP ex.: []\n",
      "FN ex.: ['ה ליגה ל זכויות ה אדם', 'ה התאחדות ל עולם טוב יותר']\n",
      "((), 7)\n",
      "1 mentions, 1 found, 1 correct.\n",
      "Precision: 1.0\n",
      "Recall:    1.0\n",
      "F1:        1.0\n",
      "FP ex.: []\n",
      "FN ex.: []\n",
      "((), 10)\n",
      "1 mentions, 0 found, 0 correct.\n",
      "Precision: -1\n",
      "Recall:    0.0\n",
      "F1:        0.0\n",
      "FP ex.: []\n",
      "FN ex.: ['מרכז ה מידע ל זכויות ה אדם ב ה שטחים']\n",
      "(('T',), 1)\n",
      "16 mentions, 12 found, 12 correct.\n",
      "Precision: 1.0\n",
      "Recall:    0.75\n",
      "F1:        0.86\n",
      "FP ex.: []\n",
      "FN ex.: ['קדיש', 'מסצוסטס', 'ארה\"ב', 'לין']\n",
      "(('T',), 2)\n",
      "7 mentions, 8 found, 6 correct.\n",
      "Precision: 0.75\n",
      "Recall:    0.86\n",
      "F1:        0.8\n",
      "FP ex.: ['ה ארה\"ב', 'ה מסצוסטס']\n",
      "FN ex.: ['עין כרם']\n",
      "(('T',), 3)\n",
      "7 mentions, 4 found, 3 correct.\n",
      "Precision: 0.75\n",
      "Recall:    0.43\n",
      "F1:        0.55\n",
      "FP ex.: ['ישיבת ה רעיון']\n",
      "FN ex.: ['תנועת ה מושבים', 'תנועת ה מושבים', 'תנועת ה מושבים', 'תנועת ה מושבים']\n",
      "(('T',), 4)\n",
      "9 mentions, 0 found, 0 correct.\n",
      "Precision: -1\n",
      "Recall:    0.0\n",
      "F1:        0.0\n",
      "FP ex.: []\n",
      "FN ex.: ['ה בית ה לבן', 'ה בית ה לבן', 'ה מלחמה ה קרה', 'ה בית ה לבן', 'ה מצביע ה אמריקאי']\n",
      "(('T',), 5)\n",
      "6 mentions, 2 found, 2 correct.\n",
      "Precision: 1.0\n",
      "Recall:    0.33\n",
      "F1:        0.5\n",
      "FP ex.: []\n",
      "FN ex.: ['ה קרב על סן סימון', 'ישיבת ה רעיון ה יהודי', 'ה קרב על סן סימון', 'ה עבודה ו ה רווחה']\n",
      "(('T', 'TM'), 2)\n",
      "3 mentions, 4 found, 3 correct.\n",
      "Precision: 0.75\n",
      "Recall:    1.0\n",
      "F1:        0.86\n",
      "FP ex.: ['שכונת קטמון']\n",
      "FN ex.: []\n",
      "(('T', 'TM'), 4)\n",
      "1 mentions, 1 found, 0 correct.\n",
      "Precision: 0.0\n",
      "Recall:    0.0\n",
      "F1:        -1\n",
      "FP ex.: ['ארגון נפגעי ה משכנתאות']\n",
      "FN ex.: ['נאום אימפריית ה רשע']\n",
      "(('T', 'TM'), 5)\n",
      "0 mentions, 1 found, 0 correct.\n",
      "Precision: 0.0\n",
      "Recall:    -1\n",
      "F1:        0.0\n",
      "FP ex.: ['כביש באב אל - ואד']\n",
      "FN ex.: []\n",
      "(('T', 'TM'), 6)\n",
      "1 mentions, 0 found, 0 correct.\n",
      "Precision: -1\n",
      "Recall:    0.0\n",
      "F1:        0.0\n",
      "FP ex.: []\n",
      "FN ex.: ['ה קרב על מנזר סן סימון']\n",
      "(('T', 'TM'), 8)\n",
      "1 mentions, 0 found, 0 correct.\n",
      "Precision: -1\n",
      "Recall:    0.0\n",
      "F1:        0.0\n",
      "FP ex.: []\n",
      "FN ex.: ['ארגון נפגעי ה משכנתאות ו חסרי ה דיור']\n",
      "(('TM',), 1)\n",
      "102 mentions, 87 found, 79 correct.\n",
      "Precision: 0.91\n",
      "Recall:    0.77\n",
      "F1:        0.84\n",
      "FP ex.: ['פראפראז', 'קטמון', 'ירסקו', 'לינד', 'גאנט']\n",
      "FN ex.: ['מפם', 'ליון', 'טובייה', 'ורמונט', 'לאקספרס']\n",
      "(('TM',), 2)\n",
      "75 mentions, 71 found, 61 correct.\n",
      "Precision: 0.86\n",
      "Recall:    0.81\n",
      "F1:        0.84\n",
      "FP ex.: ['ל אקספרס', 'ה קטיף', 'איסט סייד', 'ה קטיף', 'ה קטיף']\n",
      "FN ex.: ['חברה קדישא', 'חברה קדישא', 'טום דאיין', 'ה גסטאפו', 'ה פלמ\"ח']\n",
      "(('TM',), 3)\n",
      "18 mentions, 17 found, 9 correct.\n",
      "Precision: 0.53\n",
      "Recall:    0.5\n",
      "F1:        0.51\n",
      "FP ex.: ['מנזר סן סימון', 'סמית - ריצרדסון', 'מכון ה דסון', 'נווה - מונסון', 'אמריקן אקונומיק ריוויו']\n",
      "FN ex.: ['אמריקן אקונומיק ריוויו', 'נווה - מונסון', 'מנזר סן סימון', 'מנזר סן סימון', 'מלאך ה מוות']\n",
      "(('TM',), 4)\n",
      "6 mentions, 2 found, 2 correct.\n",
      "Precision: 1.0\n",
      "Recall:    0.33\n",
      "F1:        0.5\n",
      "FP ex.: []\n",
      "FN ex.: ['מענק \" גאונות \"', 'באב אל - ואד', 'לינד ו הרי בראדלו', 'איסט סייד ה תחתית']\n",
      "(('TM',), 5)\n",
      "0 mentions, 1 found, 0 correct.\n",
      "Precision: 0.0\n",
      "Recall:    -1\n",
      "F1:        0.0\n",
      "FP ex.: ['אייפא\"ק טום דאיין את הארקין']\n",
      "FN ex.: []\n",
      "(('TM',), 6)\n",
      "2 mentions, 0 found, 0 correct.\n",
      "Precision: -1\n",
      "Recall:    0.0\n",
      "F1:        0.0\n",
      "FP ex.: []\n",
      "FN ex.: ['ה שדולה ה פרו - ישראלית', 'ה קונסוליה ה ישראלית ב שיקאגו']\n",
      "(('TM',), 14)\n",
      "1 mentions, 0 found, 0 correct.\n",
      "Precision: -1\n",
      "Recall:    0.0\n",
      "F1:        0.0\n",
      "FP ex.: []\n",
      "FN ex.: ['ה ברוקרים של ה רעיונות : צוותות חשיבה ו עלייתה של עלית מדיניות חדשה']\n"
     ]
    }
   ],
   "source": [
    "for k in sorted(ment_set.keys() | gold_ment_set.keys()):\n",
    "    print(k)\n",
    "    nem.evaluate_mentions(gold_ment_set[k], ment_set[k])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Length+unk_type+morpheme_count"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 251,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict_keys([((('T',), 2), 1), (((), 2), 1), (((), 3), 6), (((), 2), 2), (((), 1), 2), (((), 1), 1), ((('T',), 1), 2), ((('T',), 2), 3), (((), 2), 3), ((('T',), 2), 2), (((), 2), 4), (((), 3), 5), ((('TM',), 1), 2), (((), 3), 2), ((('T', 'TM'), 1), 2), ((('TM',), 1), 1), ((('TM',), 1), 3), ((('TM',), 2), 3), ((('T', 'TM'), 1), 5), (((), 2), 5), ((('T', 'TM'), 2), 4), (((), 3), 4), ((('TM',), 2), 1), ((('TM',), 2), 2), (((), 2), 7), ((('TM',), 1), 5), ((('TM',), 3), 2), ((('T',), 2), 5), (((), 1), 3), ((('T', 'TM'), 2), 2), ((('T',), 3), 1), ((('TM',), 1), 4), ((('T',), 3), 2), (((), 1), 4)])"
      ]
     },
     "execution_count": 251,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "token_unk_sents = full_samp.groupby('sent_id').apply(get_sent, ['form', 'biose_layer0', 'unk_type', 'morpheme_count'])\n",
    "token_unk_sents_fixed = get_sents_fixed(token_unk_sents)\n",
    "ments = [sent_to_mentions_dict(sent, sent_id, group_maker=gm, ment_len=True) for sent_id, sent in token_unk_sents_fixed]\n",
    "ment_set = get_ment_set(ments)\n",
    "ment_set.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 252,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict_keys([((('T',), 2), 1), (((), 2), 1), (((), 3), 6), (((), 2), 2), (((), 1), 2), (((), 1), 1), ((('T',), 3), 5), ((('T',), 2), 3), ((('T',), 1), 2), (((), 2), 3), ((('T',), 2), 2), (((), 2), 4), (((), 3), 5), ((('TM',), 1), 2), ((('TM',), 1), 1), (((), 3), 2), ((('T',), 2), 5), (((), 3), 4), ((('TM',), 1), 3), (((), 1), 4), ((('T', 'TM'), 3), 6), ((('TM',), 2), 3), ((('TM',), 2), 2), ((('TM',), 1), 4), (((), 2), 5), ((('T', 'TM'), 2), 8), ((('TM',), 2), 1), (((), 2), 7), ((('T',), 2), 4), ((('TM',), 2), 6), ((('TM',), 3), 6), (((), 3), 10), ((('TM',), 2), 4), (((), 2), 6), (((), 1), 3), ((('T', 'TM'), 2), 2), ((('T',), 3), 1), ((('TM',), 3), 4), ((('T', 'TM'), 2), 4), ((('T',), 3), 4), ((('TM',), 2), 14)])"
      ]
     },
     "execution_count": 252,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gold_dev_sents = dev_gold.groupby('sent_id').apply(get_sent, ['form', 'biose_layer0', 'unk_type', 'morpheme_count'])\n",
    "gold_dev_sents_fixed = get_sents_fixed(gold_dev_sents)\n",
    "gold_ments = [sent_to_mentions_dict(sent, sent_id, group_maker=gm, ment_len=True) for sent_id, sent in gold_dev_sents_fixed]\n",
    "gold_ment_set = get_ment_set(gold_ments)\n",
    "gold_ment_set.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 253,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{(((), 1), 1),\n",
       " (((), 1), 2),\n",
       " (((), 1), 3),\n",
       " (((), 1), 4),\n",
       " (((), 2), 1),\n",
       " (((), 2), 2),\n",
       " (((), 2), 3),\n",
       " (((), 2), 4),\n",
       " (((), 2), 5),\n",
       " (((), 2), 6),\n",
       " (((), 2), 7),\n",
       " (((), 3), 2),\n",
       " (((), 3), 4),\n",
       " (((), 3), 5),\n",
       " (((), 3), 6),\n",
       " (((), 3), 10),\n",
       " ((('T',), 1), 2),\n",
       " ((('T',), 2), 1),\n",
       " ((('T',), 2), 2),\n",
       " ((('T',), 2), 3),\n",
       " ((('T',), 2), 4),\n",
       " ((('T',), 2), 5),\n",
       " ((('T',), 3), 1),\n",
       " ((('T',), 3), 2),\n",
       " ((('T',), 3), 4),\n",
       " ((('T',), 3), 5),\n",
       " ((('T', 'TM'), 1), 2),\n",
       " ((('T', 'TM'), 1), 5),\n",
       " ((('T', 'TM'), 2), 2),\n",
       " ((('T', 'TM'), 2), 4),\n",
       " ((('T', 'TM'), 2), 8),\n",
       " ((('T', 'TM'), 3), 6),\n",
       " ((('TM',), 1), 1),\n",
       " ((('TM',), 1), 2),\n",
       " ((('TM',), 1), 3),\n",
       " ((('TM',), 1), 4),\n",
       " ((('TM',), 1), 5),\n",
       " ((('TM',), 2), 1),\n",
       " ((('TM',), 2), 2),\n",
       " ((('TM',), 2), 3),\n",
       " ((('TM',), 2), 4),\n",
       " ((('TM',), 2), 6),\n",
       " ((('TM',), 2), 14),\n",
       " ((('TM',), 3), 2),\n",
       " ((('TM',), 3), 4),\n",
       " ((('TM',), 3), 6)}"
      ]
     },
     "execution_count": 253,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ment_set.keys() | gold_ment_set.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 254,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(((), 1), 1)\n",
      "73 mentions, 68 found, 56 correct.\n",
      "Precision: 0.82\n",
      "Recall:    0.77\n",
      "F1:        0.79\n",
      "FP ex.: ['מסצוסטס', 'גרוסבורד', 'הראל', 'מסצוסטס', 'מסצוסטס']\n",
      "FN ex.: ['ה', 'כך', 'וייטנאם', 'מסצוסטס', 'פיו']\n",
      "(((), 1), 2)\n",
      "39 mentions, 42 found, 36 correct.\n",
      "Precision: 0.86\n",
      "Recall:    0.92\n",
      "F1:        0.89\n",
      "FP ex.: ['מחוז פאריס', 'קרן גון', 'סן סימון', 'נא מילוא', 'שלום עכשיו']\n",
      "FN ex.: ['מחוז פאריס', 'בית נבחרים', 'סנטר 1']\n",
      "(((), 1), 3)\n",
      "4 mentions, 3 found, 3 correct.\n",
      "Precision: 1.0\n",
      "Recall:    0.75\n",
      "F1:        0.86\n",
      "FP ex.: []\n",
      "FN ex.: ['בית מספר 3']\n",
      "(((), 1), 4)\n",
      "2 mentions, 1 found, 1 correct.\n",
      "Precision: 1.0\n",
      "Recall:    0.5\n",
      "F1:        0.67\n",
      "FP ex.: []\n",
      "FN ex.: ['חטיבת \" הראל \"']\n",
      "(((), 2), 1)\n",
      "37 mentions, 40 found, 36 correct.\n",
      "Precision: 0.9\n",
      "Recall:    0.97\n",
      "F1:        0.94\n",
      "FP ex.: ['ירושלים', 'מקורות', 'אושוויץ', 'שיקאגו']\n",
      "FN ex.: ['אושוויץ']\n",
      "(((), 2), 2)\n",
      "19 mentions, 17 found, 9 correct.\n",
      "Precision: 0.53\n",
      "Recall:    0.47\n",
      "F1:        0.5\n",
      "FP ex.: ['ה שמרנים', 'ה ארץ', 'ה שמרנים', 'ה ארץ', 'ה קב\"ה']\n",
      "FN ex.: ['ניו גרסי', 'ה גרמנים', 'ה מלחמה', 'ה קב\"ה', 'ה משטרה']\n",
      "(((), 2), 3)\n",
      "19 mentions, 20 found, 15 correct.\n",
      "Precision: 0.75\n",
      "Recall:    0.79\n",
      "F1:        0.77\n",
      "FP ex.: ['רצח ה נזירות', 'מדינות ה ברית', 'רחוב גבעת שאול', 'גורג ה מסכן', 'ה שירות פועל']\n",
      "FN ex.: ['ירושלים ה מערבית', 'מלחמת ה עצמאות', 'רחוב גבעת שאול', 'ניו - יורק']\n",
      "(((), 2), 4)\n",
      "19 mentions, 19 found, 15 correct.\n",
      "Precision: 0.79\n",
      "Recall:    0.79\n",
      "F1:        0.79\n",
      "FP ex.: ['הר - ה בית', 'שכונת שמואל ה נביא', 'שלום ה רב כהנא', 'ה מלחמת ה עצמאות']\n",
      "FN ex.: ['הר - ה בית', 'שכונת שמואל ה נביא', 'פרס נובל ל שלום', 'ה עולם ה שלישי']\n",
      "(((), 2), 5)\n",
      "5 mentions, 5 found, 4 correct.\n",
      "Precision: 0.8\n",
      "Recall:    0.8\n",
      "F1:        0.8\n",
      "FP ex.: ['ישיבת ה רעיון ה יהודי']\n",
      "FN ex.: ['ישיבת ה רעיון ה יהודי']\n",
      "(((), 2), 6)\n",
      "2 mentions, 0 found, 0 correct.\n",
      "Precision: -1\n",
      "Recall:    0.0\n",
      "F1:        0.0\n",
      "FP ex.: []\n",
      "FN ex.: ['ה ליגה ל זכויות ה אדם', 'ה התאחדות ל עולם טוב יותר']\n",
      "(((), 2), 7)\n",
      "1 mentions, 1 found, 1 correct.\n",
      "Precision: 1.0\n",
      "Recall:    1.0\n",
      "F1:        1.0\n",
      "FP ex.: []\n",
      "FN ex.: []\n",
      "(((), 3), 2)\n",
      "11 mentions, 10 found, 8 correct.\n",
      "Precision: 0.8\n",
      "Recall:    0.73\n",
      "F1:        0.76\n",
      "FP ex.: ['ה שטחים', 'ה ארץ']\n",
      "FN ex.: ['ה ארץ', 'ה ארץ', 'ה כנסת']\n",
      "(((), 3), 4)\n",
      "8 mentions, 6 found, 6 correct.\n",
      "Precision: 1.0\n",
      "Recall:    0.75\n",
      "F1:        0.86\n",
      "FP ex.: []\n",
      "FN ex.: ['ה עולם ה שלישי', 'ה קרב על ירושלים']\n",
      "(((), 3), 5)\n",
      "1 mentions, 1 found, 1 correct.\n",
      "Precision: 1.0\n",
      "Recall:    1.0\n",
      "F1:        1.0\n",
      "FP ex.: []\n",
      "FN ex.: []\n",
      "(((), 3), 6)\n",
      "3 mentions, 3 found, 3 correct.\n",
      "Precision: 1.0\n",
      "Recall:    1.0\n",
      "F1:        1.0\n",
      "FP ex.: []\n",
      "FN ex.: []\n",
      "(((), 3), 10)\n",
      "1 mentions, 0 found, 0 correct.\n",
      "Precision: -1\n",
      "Recall:    0.0\n",
      "F1:        0.0\n",
      "FP ex.: []\n",
      "FN ex.: ['מרכז ה מידע ל זכויות ה אדם ב ה שטחים']\n",
      "((('T',), 1), 2)\n",
      "2 mentions, 2 found, 2 correct.\n",
      "Precision: 1.0\n",
      "Recall:    1.0\n",
      "F1:        1.0\n",
      "FP ex.: []\n",
      "FN ex.: []\n",
      "((('T',), 2), 1)\n",
      "12 mentions, 10 found, 10 correct.\n",
      "Precision: 1.0\n",
      "Recall:    0.83\n",
      "F1:        0.91\n",
      "FP ex.: []\n",
      "FN ex.: ['קדיש', 'לין']\n",
      "((('T',), 2), 2)\n",
      "5 mentions, 4 found, 4 correct.\n",
      "Precision: 1.0\n",
      "Recall:    0.8\n",
      "F1:        0.89\n",
      "FP ex.: []\n",
      "FN ex.: ['עין כרם']\n",
      "((('T',), 2), 3)\n",
      "7 mentions, 4 found, 3 correct.\n",
      "Precision: 0.75\n",
      "Recall:    0.43\n",
      "F1:        0.55\n",
      "FP ex.: ['ישיבת ה רעיון']\n",
      "FN ex.: ['תנועת ה מושבים', 'תנועת ה מושבים', 'תנועת ה מושבים', 'תנועת ה מושבים']\n",
      "((('T',), 2), 4)\n",
      "5 mentions, 0 found, 0 correct.\n",
      "Precision: -1\n",
      "Recall:    0.0\n",
      "F1:        0.0\n",
      "FP ex.: []\n",
      "FN ex.: ['ה מלחמה ה קרה', 'ה בית ה לבן', 'ה בית ה לבן', 'צדקה מתחילה מ בית', 'ה מצביע ה אמריקאי']\n",
      "((('T',), 2), 5)\n",
      "5 mentions, 2 found, 2 correct.\n",
      "Precision: 1.0\n",
      "Recall:    0.4\n",
      "F1:        0.57\n",
      "FP ex.: []\n",
      "FN ex.: ['ה קרב על סן סימון', 'ישיבת ה רעיון ה יהודי', 'ה קרב על סן סימון']\n",
      "((('T',), 3), 1)\n",
      "4 mentions, 2 found, 2 correct.\n",
      "Precision: 1.0\n",
      "Recall:    0.5\n",
      "F1:        0.67\n",
      "FP ex.: []\n",
      "FN ex.: ['מסצוסטס', 'ארה\"ב']\n",
      "((('T',), 3), 2)\n",
      "0 mentions, 2 found, 0 correct.\n",
      "Precision: 0.0\n",
      "Recall:    -1\n",
      "F1:        0.0\n",
      "FP ex.: ['ה ארה\"ב', 'ה מסצוסטס']\n",
      "FN ex.: []\n",
      "((('T',), 3), 4)\n",
      "4 mentions, 0 found, 0 correct.\n",
      "Precision: -1\n",
      "Recall:    0.0\n",
      "F1:        0.0\n",
      "FP ex.: []\n",
      "FN ex.: ['ה מלחמה ה קרה', 'ה בית ה לבן', 'ה בית ה לבן', 'ה מלחמה ה קרה']\n",
      "((('T',), 3), 5)\n",
      "1 mentions, 0 found, 0 correct.\n",
      "Precision: -1\n",
      "Recall:    0.0\n",
      "F1:        0.0\n",
      "FP ex.: []\n",
      "FN ex.: ['ה עבודה ו ה רווחה']\n",
      "((('T', 'TM'), 1), 2)\n",
      "0 mentions, 1 found, 0 correct.\n",
      "Precision: 0.0\n",
      "Recall:    -1\n",
      "F1:        0.0\n",
      "FP ex.: ['שכונת קטמון']\n",
      "FN ex.: []\n",
      "((('T', 'TM'), 1), 5)\n",
      "0 mentions, 1 found, 0 correct.\n",
      "Precision: 0.0\n",
      "Recall:    -1\n",
      "F1:        0.0\n",
      "FP ex.: ['כביש באב אל - ואד']\n",
      "FN ex.: []\n",
      "((('T', 'TM'), 2), 2)\n",
      "3 mentions, 3 found, 3 correct.\n",
      "Precision: 1.0\n",
      "Recall:    1.0\n",
      "F1:        1.0\n",
      "FP ex.: []\n",
      "FN ex.: []\n",
      "((('T', 'TM'), 2), 4)\n",
      "1 mentions, 1 found, 0 correct.\n",
      "Precision: 0.0\n",
      "Recall:    0.0\n",
      "F1:        -1\n",
      "FP ex.: ['ארגון נפגעי ה משכנתאות']\n",
      "FN ex.: ['נאום אימפריית ה רשע']\n",
      "((('T', 'TM'), 2), 8)\n",
      "1 mentions, 0 found, 0 correct.\n",
      "Precision: -1\n",
      "Recall:    0.0\n",
      "F1:        0.0\n",
      "FP ex.: []\n",
      "FN ex.: ['ארגון נפגעי ה משכנתאות ו חסרי ה דיור']\n",
      "((('T', 'TM'), 3), 6)\n",
      "1 mentions, 0 found, 0 correct.\n",
      "Precision: -1\n",
      "Recall:    0.0\n",
      "F1:        0.0\n",
      "FP ex.: []\n",
      "FN ex.: ['ה קרב על מנזר סן סימון']\n",
      "((('TM',), 1), 1)\n",
      "79 mentions, 67 found, 59 correct.\n",
      "Precision: 0.88\n",
      "Recall:    0.75\n",
      "F1:        0.81\n",
      "FP ex.: ['פראפראז', 'קטמון', 'ירסקו', 'לינד', 'גאנט']\n",
      "FN ex.: ['מפם', 'טובייה', 'ורמונט', 'לאקספרס', 'מקארתור']\n",
      "((('TM',), 1), 2)\n",
      "62 mentions, 61 found, 57 correct.\n",
      "Precision: 0.93\n",
      "Recall:    0.92\n",
      "F1:        0.93\n",
      "FP ex.: ['ל אקספרס', 'מחוז זירונד', 'דה מוין', 'שרה סקאיף']\n",
      "FN ex.: ['טום דאיין', 'שרה סקאיף', 'דה מוין', 'מחוז זירונד', 'מכון הדסון']\n",
      "((('TM',), 1), 3)\n",
      "11 mentions, 12 found, 7 correct.\n",
      "Precision: 0.58\n",
      "Recall:    0.64\n",
      "F1:        0.61\n",
      "FP ex.: ['מנזר סן סימון', 'סמית - ריצרדסון', 'מכון ה דסון', 'אמריקן אקונומיק ריוויו', 'גנאדאס דל ואיה']\n",
      "FN ex.: ['גנאדאס דל ואיה', 'סמית - ריצרדסון', 'אמריקן אקונומיק ריוויו', 'מנזר סן סימון']\n",
      "((('TM',), 1), 4)\n",
      "3 mentions, 2 found, 2 correct.\n",
      "Precision: 1.0\n",
      "Recall:    0.67\n",
      "F1:        0.8\n",
      "FP ex.: []\n",
      "FN ex.: ['באב אל - ואד']\n",
      "((('TM',), 1), 5)\n",
      "0 mentions, 1 found, 0 correct.\n",
      "Precision: 0.0\n",
      "Recall:    -1\n",
      "F1:        0.0\n",
      "FP ex.: ['אייפא\"ק טום דאיין את הארקין']\n",
      "FN ex.: []\n",
      "((('TM',), 2), 1)\n",
      "23 mentions, 20 found, 20 correct.\n",
      "Precision: 1.0\n",
      "Recall:    0.87\n",
      "F1:        0.93\n",
      "FP ex.: []\n",
      "FN ex.: ['קריסטול', 'ליון', 'אלוקים']\n",
      "((('TM',), 2), 2)\n",
      "13 mentions, 5 found, 4 correct.\n",
      "Precision: 0.8\n",
      "Recall:    0.31\n",
      "F1:        0.44\n",
      "FP ex.: ['הרי בראדלו']\n",
      "FN ex.: ['חברה קדישא', 'חברה קדישא', 'ה גסטאפו', 'ה פלמ\"ח', 'קרן פורד']\n",
      "((('TM',), 2), 3)\n",
      "7 mentions, 5 found, 2 correct.\n",
      "Precision: 0.4\n",
      "Recall:    0.29\n",
      "F1:        0.33\n",
      "FP ex.: ['נווה - מונסון', 'קרן פורד שמרנים', 'קטיף ה הדרים']\n",
      "FN ex.: ['נווה - מונסון', 'מנזר סן סימון', 'מלאך ה מוות', 'תיאוריית ה יחסות', 'תיאוריית ה קוונטים']\n",
      "((('TM',), 2), 4)\n",
      "2 mentions, 0 found, 0 correct.\n",
      "Precision: -1\n",
      "Recall:    0.0\n",
      "F1:        0.0\n",
      "FP ex.: []\n",
      "FN ex.: ['מענק \" גאונות \"', 'לינד ו הרי בראדלו']\n",
      "((('TM',), 2), 6)\n",
      "1 mentions, 0 found, 0 correct.\n",
      "Precision: -1\n",
      "Recall:    0.0\n",
      "F1:        0.0\n",
      "FP ex.: []\n",
      "FN ex.: ['ה שדולה ה פרו - ישראלית']\n",
      "((('TM',), 2), 14)\n",
      "1 mentions, 0 found, 0 correct.\n",
      "Precision: -1\n",
      "Recall:    0.0\n",
      "F1:        0.0\n",
      "FP ex.: []\n",
      "FN ex.: ['ה ברוקרים של ה רעיונות : צוותות חשיבה ו עלייתה של עלית מדיניות חדשה']\n",
      "((('TM',), 3), 2)\n",
      "0 mentions, 5 found, 0 correct.\n",
      "Precision: 0.0\n",
      "Recall:    -1\n",
      "F1:        0.0\n",
      "FP ex.: ['ה קטיף', 'ה קטיף', 'ה קטיף', 'ה קטיף', 'איסט סייד']\n",
      "FN ex.: []\n",
      "((('TM',), 3), 4)\n",
      "1 mentions, 0 found, 0 correct.\n",
      "Precision: -1\n",
      "Recall:    0.0\n",
      "F1:        0.0\n",
      "FP ex.: []\n",
      "FN ex.: ['איסט סייד ה תחתית']\n",
      "((('TM',), 3), 6)\n",
      "1 mentions, 0 found, 0 correct.\n",
      "Precision: -1\n",
      "Recall:    0.0\n",
      "F1:        0.0\n",
      "FP ex.: []\n",
      "FN ex.: ['ה קונסוליה ה ישראלית ב שיקאגו']\n"
     ]
    }
   ],
   "source": [
    "for k in sorted(ment_set.keys() | gold_ment_set.keys()):\n",
    "    print(k)\n",
    "    nem.evaluate_mentions(gold_ment_set[k], ment_set[k])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Check OOEV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 269,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "False    11127\n",
       "True       174\n",
       "Name: ooev, dtype: int64"
      ]
     },
     "execution_count": 269,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "full_samp['ooev'] = ~full_samp.form.isin(emb_words['tok'])\n",
    "full_samp.ooev.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 270,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "False    8357\n",
       "True      174\n",
       "Name: ooev_tok_part, dtype: int64"
      ]
     },
     "execution_count": 270,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ooev_tok_part = (full_samp.groupby(['sent_id', 'token_id'])\n",
    "                 .ooev.apply(lambda x: x.any())\n",
    "                 .reset_index().rename(columns={'ooev': 'ooev_tok_part'}))\n",
    "ooev_tok_part.ooev_tok_part.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 271,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((11301, 22), (11301, 34))"
      ]
     },
     "execution_count": 271,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "full_samp = full_samp.merge(ooev_tok_part, how='left')\n",
    "dev_gold = dev_gold.merge(ooev_tok_part, how='left')\n",
    "full_samp.shape, dev_gold.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 272,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict_keys([(False,), (False, True), (True,)])"
      ]
     },
     "execution_count": 272,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "token_unk_sents = full_samp.groupby('sent_id').apply(get_sent, ['form', 'biose_layer0', 'ooev_tok_part'])\n",
    "token_unk_sents_fixed = get_sents_fixed(token_unk_sents)\n",
    "ments = [sent_to_mentions_dict(sent, sent_id) for sent_id, sent in token_unk_sents_fixed]\n",
    "ment_set = get_ment_set(ments)\n",
    "ment_set.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 273,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict_keys([(False,), (False, True), (True,)])"
      ]
     },
     "execution_count": 273,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gold_dev_sents = dev_gold.groupby('sent_id').apply(get_sent, ['form', 'biose_layer0', 'ooev_tok_part'])\n",
    "gold_dev_sents_fixed = get_sents_fixed(gold_dev_sents)\n",
    "gold_ments = [sent_to_mentions_dict(sent, sent_id) for sent_id, sent in gold_dev_sents_fixed]\n",
    "gold_ment_set = get_ment_set(gold_ments)\n",
    "gold_ment_set.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 274,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{(False,), (False, True), (True,)}"
      ]
     },
     "execution_count": 274,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ment_set.keys() | gold_ment_set.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 275,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(False, True)\n",
      "24 mentions, 24 found, 17 correct.\n",
      "Precision: 0.71\n",
      "Recall:    0.71\n",
      "F1:        0.71\n",
      "FP ex.: ['סמית - ריצרדסון', 'אייפא\"ק טום דאיין את הארקין', 'אמריקן אקונומיק ריוויו', 'הרי בראדלו', 'מחוז זירונד']\n",
      "FN ex.: ['אמריקן אקונומיק ריוויו', 'לינד ו הרי בראדלו', 'שרה סקאיף', 'גנאדאס דל ואיה', 'סמית - ריצרדסון']\n",
      "(False,)\n",
      "441 mentions, 389 found, 330 correct.\n",
      "Precision: 0.85\n",
      "Recall:    0.75\n",
      "F1:        0.8\n",
      "FP ex.: ['שלום ה רב כהנא', 'קרן גון', 'סן סימון', 'ל אקספרס', 'קטמון']\n",
      "FN ex.: ['ה ליגה ל זכויות ה אדם', 'טום דאיין', 'טובייה', 'ה ברוקרים של ה רעיונות : צוותות חשיבה ו עלייתה של עלית מדיניות חדשה', 'ה קרב על מנזר סן סימון']\n",
      "(True,)\n",
      "34 mentions, 33 found, 24 correct.\n",
      "Precision: 0.73\n",
      "Recall:    0.71\n",
      "F1:        0.72\n",
      "FP ex.: ['מסצוסטס', 'גרוסבורד', 'פראפראז', 'מסצוסטס', 'מסצוסטס']\n",
      "FN ex.: ['מסצוסטס', 'נבראסקה', 'מסצוסטס', 'ה קב\"ה', 'אייפא\"ק']\n"
     ]
    }
   ],
   "source": [
    "for k in ment_set.keys() | gold_ment_set.keys():\n",
    "    print(k)\n",
    "    nem.evaluate_mentions(gold_ment_set[k], ment_set[k])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## OOEV + unk_type"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 281,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict_keys([(('T',), False), ((), False), (('TM',), False), (('T', 'TM'), False), (('TM',), True), ((), True), (('T',), True), (('T', 'TM'), True)])"
      ]
     },
     "execution_count": 281,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "token_unk_sents = full_samp.groupby('sent_id').apply(get_sent, ['form', 'biose_layer0', 'unk_type', 'ooev_tok_part'])\n",
    "token_unk_sents_fixed = get_sents_fixed(token_unk_sents)\n",
    "ments = [sent_to_mentions_dict(sent, sent_id, group_maker=gm) for sent_id, sent in token_unk_sents_fixed]\n",
    "ment_set = get_ment_set(ments)\n",
    "ment_set.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 280,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict_keys([(('T',), False), ((), False), (('TM',), False), (('T', 'TM'), False), (('TM',), True), ((), True), (('T',), True), (('T', 'TM'), True)])"
      ]
     },
     "execution_count": 280,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gold_dev_sents = dev_gold.groupby('sent_id').apply(get_sent, ['form', 'biose_layer0', 'unk_type', 'ooev_tok_part'])\n",
    "gold_dev_sents_fixed = get_sents_fixed(gold_dev_sents)\n",
    "gold_ments = [sent_to_mentions_dict(sent, sent_id, group_maker=gm) for sent_id, sent in gold_dev_sents_fixed]\n",
    "gold_ment_set = get_ment_set(gold_ments)\n",
    "gold_ment_set.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 282,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{((), False),\n",
       " ((), True),\n",
       " (('T',), False),\n",
       " (('T',), True),\n",
       " (('T', 'TM'), False),\n",
       " (('T', 'TM'), True),\n",
       " (('TM',), False),\n",
       " (('TM',), True)}"
      ]
     },
     "execution_count": 282,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ment_set.keys() | gold_ment_set.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 283,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(('TM',), False)\n",
      "170 mentions, 146 found, 128 correct.\n",
      "Precision: 0.88\n",
      "Recall:    0.75\n",
      "F1:        0.81\n",
      "FP ex.: ['מנזר סן סימון', 'מכון ה דסון', 'נווה - מונסון', 'ל אקספרס', 'קטמון']\n",
      "FN ex.: ['חברה קדישא', 'נווה - מונסון', 'מפם', 'טום דאיין', 'ה שדולה ה פרו - ישראלית']\n",
      "(('T', 'TM'), True)\n",
      "1 mentions, 1 found, 1 correct.\n",
      "Precision: 1.0\n",
      "Recall:    1.0\n",
      "F1:        1.0\n",
      "FP ex.: []\n",
      "FN ex.: []\n",
      "(('T', 'TM'), False)\n",
      "5 mentions, 5 found, 2 correct.\n",
      "Precision: 0.4\n",
      "Recall:    0.4\n",
      "F1:        0.4\n",
      "FP ex.: ['שכונת קטמון', 'ארגון נפגעי ה משכנתאות', 'כביש באב אל - ואד']\n",
      "FN ex.: ['נאום אימפריית ה רשע', 'ארגון נפגעי ה משכנתאות ו חסרי ה דיור', 'ה קרב על מנזר סן סימון']\n",
      "(('T',), False)\n",
      "43 mentions, 24 found, 22 correct.\n",
      "Precision: 0.92\n",
      "Recall:    0.51\n",
      "F1:        0.66\n",
      "FP ex.: ['ישיבת ה רעיון', 'ה ארה\"ב']\n",
      "FN ex.: ['ארה\"ב', 'תנועת ה מושבים', 'ה מצביע ה אמריקאי', 'ה קרב על סן סימון', 'תנועת ה מושבים']\n",
      "(('T',), True)\n",
      "2 mentions, 2 found, 1 correct.\n",
      "Precision: 0.5\n",
      "Recall:    0.5\n",
      "F1:        0.5\n",
      "FP ex.: ['ה מסצוסטס']\n",
      "FN ex.: ['מסצוסטס']\n",
      "((), False)\n",
      "223 mentions, 214 found, 178 correct.\n",
      "Precision: 0.83\n",
      "Recall:    0.8\n",
      "F1:        0.81\n",
      "FP ex.: ['שלום ה רב כהנא', 'קרן גון', 'ה שטחים', 'ה שמרנים', 'סן סימון']\n",
      "FN ex.: ['ה ליגה ל זכויות ה אדם', 'פיו', 'ה התאחדות ל עולם טוב יותר', 'פיו', 'אושוויץ']\n",
      "((), True)\n",
      "21 mentions, 22 found, 16 correct.\n",
      "Precision: 0.73\n",
      "Recall:    0.76\n",
      "F1:        0.74\n",
      "FP ex.: ['מסצוסטס', 'גרוסבורד', 'מסצוסטס', 'מסצוסטס', 'שיקאגו']\n",
      "FN ex.: ['מסצוסטס', 'ה קב\"ה', 'מסצוסטס', 'ה כנסת', 'מסצוסטס']\n",
      "(('TM',), True)\n",
      "34 mentions, 32 found, 23 correct.\n",
      "Precision: 0.72\n",
      "Recall:    0.68\n",
      "F1:        0.7\n",
      "FP ex.: ['פראפראז', 'סמית - ריצרדסון', 'ירסקו', 'אייפא\"ק טום דאיין את הארקין', 'אמריקן אקונומיק ריוויו']\n",
      "FN ex.: ['אמריקן אקונומיק ריוויו', 'לינד ו הרי בראדלו', 'נבראסקה', 'שרה סקאיף', 'אייפא\"ק']\n"
     ]
    }
   ],
   "source": [
    "for k in ment_set.keys() | gold_ment_set.keys():\n",
    "    print(k)\n",
    "    nem.evaluate_mentions(gold_ment_set[k], ment_set[k])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## EVERYTHING :OOO"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 284,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict_keys([((('T', 2, False),), 1), ((('', 2, False),), 1), ((('', 2, False), ('', 3, False)), 6), ((('', 2, False),), 2), ((('', 1, False),), 2), ((('', 1, False),), 1), ((('', 1, False), ('', 2, False), ('', 3, False)), 6), ((('', 1, False), ('T', 1, False)), 2), ((('', 2, False), ('T', 2, False)), 3), ((('', 1, False), ('', 2, False)), 3), ((('', 1, False), ('T', 2, False)), 2), ((('', 2, False),), 4), ((('', 2, False), ('', 3, False)), 5), ((('', 1, False), ('TM', 1, False)), 2), ((('', 3, False),), 2), ((('T', 1, False), ('TM', 1, False)), 2), ((('TM', 1, False),), 1), ((('', 1, False), ('TM', 1, False)), 3), ((('', 1, False), ('TM', 1, False), ('TM', 2, False)), 3), ((('TM', 1, False),), 2), ((('', 1, False), ('T', 1, False), ('TM', 1, False)), 5), ((('', 1, False), ('', 2, False)), 5), ((('TM', 1, False), ('TM', 1, True)), 2), ((('', 1, False), ('T', 2, False), ('TM', 1, False)), 4), ((('', 2, False), ('', 3, False)), 4), ((('TM', 2, False),), 1), ((('TM', 1, False), ('TM', 2, False)), 2), ((('', 1, False), ('', 2, False)), 7), ((('', 1, False), ('TM', 1, False), ('TM', 1, True)), 5), ((('TM', 2, False),), 3), ((('TM', 3, False),), 2), ((('', 1, False), ('T', 2, False)), 3), ((('', 1, False), ('', 2, False)), 4), ((('', 1, False), ('', 1, True)), 2), ((('', 1, True),), 1), ((('T', 2, True),), 1), ((('', 2, False), ('T', 2, False)), 5), ((('', 2, True),), 1), ((('', 2, False),), 3), ((('', 1, False), ('TM', 1, False), ('TM', 1, True)), 3), ((('', 1, False), ('TM', 1, True)), 2), ((('', 1, False),), 3), ((('', 1, False), ('', 2, False)), 2), ((('T', 2, False), ('TM', 1, False)), 2), ((('', 1, False), ('', 2, False), ('T', 2, False)), 5), ((('TM', 2, True),), 1), ((('T', 3, False),), 1), ((('', 1, False), ('TM', 1, True)), 3), ((('', 2, False), ('TM', 1, False)), 2), ((('TM', 1, True),), 1), ((('', 1, False), ('TM', 1, False), ('TM', 1, True)), 4), ((('', 2, False), ('TM', 1, True)), 2), ((('T', 2, False), ('TM', 1, True)), 2), ((('', 1, False), ('', 2, False), ('TM', 1, False)), 3), ((('TM', 1, False), ('TM', 3, False)), 2), ((('T', 3, True),), 2), ((('TM', 1, False),), 3), ((('', 2, False), ('T', 1, False)), 3), ((('', 2, True),), 2), ((('', 2, False),), 5), ((('', 1, False),), 4), ((('T', 3, False),), 2), ((('', 1, False), ('TM', 2, False)), 3)])"
      ]
     },
     "execution_count": 284,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "token_unk_sents = full_samp.groupby('sent_id').apply(get_sent, ['form', 'biose_layer0', 'unk_type', 'morpheme_count', 'ooev_tok_part'])\n",
    "token_unk_sents_fixed = get_sents_fixed(token_unk_sents)\n",
    "ments = [sent_to_mentions_dict(sent, sent_id, ment_len=True) for sent_id, sent in token_unk_sents_fixed]\n",
    "ment_set = get_ment_set(ments)\n",
    "ment_set.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 285,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict_keys([((('T', 2, False),), 1), ((('', 2, False),), 1), ((('', 2, False), ('', 3, False)), 6), ((('', 2, False),), 2), ((('', 1, False),), 2), ((('', 1, False),), 1), ((('', 1, False), ('', 2, False), ('', 3, False)), 6), ((('', 3, False), ('T', 3, False)), 5), ((('', 1, False), ('T', 2, False)), 3), ((('', 1, False), ('T', 1, False)), 2), ((('', 2, False), ('T', 2, False)), 3), ((('', 1, False), ('', 2, False)), 3), ((('', 1, False), ('T', 2, False)), 2), ((('', 2, False),), 4), ((('', 2, False), ('', 3, False)), 5), ((('', 1, False), ('TM', 1, False)), 2), ((('TM', 1, False),), 1), ((('', 3, False),), 2), ((('', 1, False), ('T', 2, False)), 5), ((('', 2, False),), 3), ((('', 1, False), ('', 3, False)), 4), ((('', 1, False), ('TM', 1, False)), 3), ((('', 1, False),), 4), ((('', 1, False), ('T', 3, False), ('TM', 1, False)), 6), ((('', 1, False), ('TM', 2, False)), 3), ((('TM', 2, False),), 2), ((('', 1, False), ('TM', 1, False), ('TM', 2, False)), 3), ((('TM', 1, False),), 2), ((('', 1, False), ('TM', 1, False)), 4), ((('', 1, False), ('', 2, False)), 5), ((('TM', 1, False), ('TM', 1, True)), 2), ((('', 1, False), ('', 2, False), ('T', 2, False), ('TM', 1, False)), 8), ((('', 2, False), ('', 3, False)), 4), ((('TM', 2, False),), 1), ((('', 1, False), ('', 2, False)), 2), ((('', 2, False), ('TM', 1, False)), 3), ((('TM', 1, False), ('TM', 2, False)), 2), ((('', 1, False), ('', 2, False)), 7), ((('', 2, False), ('T', 2, False)), 4), ((('', 1, False), ('', 2, False), ('TM', 2, False)), 6), ((('TM', 1, True),), 1), ((('', 1, False), ('', 2, False)), 4), ((('', 1, False), ('', 1, True)), 2), ((('', 1, True),), 1), ((('T', 2, True),), 1), ((('', 2, False), ('T', 2, False)), 5), ((('', 2, False), ('', 2, True), ('TM', 3, False)), 6), ((('', 2, True),), 1), ((('', 1, False), ('', 2, False), ('', 3, False)), 10), ((('', 1, False), ('TM', 1, False), ('TM', 2, False)), 4), ((('', 1, False), ('TM', 1, False), ('TM', 1, True)), 3), ((('', 2, False), ('TM', 1, False)), 2), ((('', 1, False), ('', 2, False)), 6), ((('', 1, False), ('TM', 1, True)), 2), ((('', 1, False),), 3), ((('', 1, False), ('', 2, False), ('T', 1, False)), 4), ((('T', 2, False), ('TM', 1, False)), 2), ((('', 1, False), ('', 2, False), ('T', 2, False)), 5), ((('TM', 2, True),), 1), ((('T', 3, False),), 1), ((('', 1, False), ('TM', 1, True)), 3), ((('', 1, False), ('TM', 1, False), ('TM', 1, True)), 4), ((('', 2, False), ('TM', 1, False), ('TM', 1, True)), 4), ((('T', 2, False), ('TM', 1, True)), 2), ((('TM', 1, False), ('TM', 2, False)), 3), ((('TM', 2, False),), 3), ((('', 2, False), ('TM', 1, False), ('TM', 3, False)), 4), ((('', 1, False), ('T', 2, False), ('TM', 2, False)), 4), ((('T', 2, False), ('T', 3, False)), 4), ((('', 1, False), ('', 2, False), ('TM', 1, False), ('TM', 2, False)), 14), ((('', 3, False), ('T', 2, False)), 4), ((('T', 3, True),), 1), ((('TM', 1, False),), 3), ((('', 2, False),), 6), ((('', 2, False), ('T', 1, False)), 5), ((('', 2, True),), 2), ((('', 2, False),), 5), ((('', 3, True),), 2)])"
      ]
     },
     "execution_count": 285,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gold_dev_sents = dev_gold.groupby('sent_id').apply(get_sent, ['form', 'biose_layer0', 'unk_type', 'morpheme_count', 'ooev_tok_part'])\n",
    "gold_dev_sents_fixed = get_sents_fixed(gold_dev_sents)\n",
    "gold_ments = [sent_to_mentions_dict(sent, sent_id, ment_len=True) for sent_id, sent in gold_dev_sents_fixed]\n",
    "gold_ment_set = get_ment_set(gold_ments)\n",
    "gold_ment_set.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 286,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{((('', 1, False),), 1),\n",
       " ((('', 1, False),), 2),\n",
       " ((('', 1, False),), 3),\n",
       " ((('', 1, False),), 4),\n",
       " ((('', 1, False), ('', 1, True)), 2),\n",
       " ((('', 1, False), ('', 2, False)), 2),\n",
       " ((('', 1, False), ('', 2, False)), 3),\n",
       " ((('', 1, False), ('', 2, False)), 4),\n",
       " ((('', 1, False), ('', 2, False)), 5),\n",
       " ((('', 1, False), ('', 2, False)), 6),\n",
       " ((('', 1, False), ('', 2, False)), 7),\n",
       " ((('', 1, False), ('', 2, False), ('', 3, False)), 6),\n",
       " ((('', 1, False), ('', 2, False), ('', 3, False)), 10),\n",
       " ((('', 1, False), ('', 2, False), ('T', 1, False)), 4),\n",
       " ((('', 1, False), ('', 2, False), ('T', 2, False)), 5),\n",
       " ((('', 1, False), ('', 2, False), ('T', 2, False), ('TM', 1, False)), 8),\n",
       " ((('', 1, False), ('', 2, False), ('TM', 1, False)), 3),\n",
       " ((('', 1, False), ('', 2, False), ('TM', 1, False), ('TM', 2, False)), 14),\n",
       " ((('', 1, False), ('', 2, False), ('TM', 2, False)), 6),\n",
       " ((('', 1, False), ('', 3, False)), 4),\n",
       " ((('', 1, False), ('T', 1, False)), 2),\n",
       " ((('', 1, False), ('T', 1, False), ('TM', 1, False)), 5),\n",
       " ((('', 1, False), ('T', 2, False)), 2),\n",
       " ((('', 1, False), ('T', 2, False)), 3),\n",
       " ((('', 1, False), ('T', 2, False)), 5),\n",
       " ((('', 1, False), ('T', 2, False), ('TM', 1, False)), 4),\n",
       " ((('', 1, False), ('T', 2, False), ('TM', 2, False)), 4),\n",
       " ((('', 1, False), ('T', 3, False), ('TM', 1, False)), 6),\n",
       " ((('', 1, False), ('TM', 1, False)), 2),\n",
       " ((('', 1, False), ('TM', 1, False)), 3),\n",
       " ((('', 1, False), ('TM', 1, False)), 4),\n",
       " ((('', 1, False), ('TM', 1, False), ('TM', 1, True)), 3),\n",
       " ((('', 1, False), ('TM', 1, False), ('TM', 1, True)), 4),\n",
       " ((('', 1, False), ('TM', 1, False), ('TM', 1, True)), 5),\n",
       " ((('', 1, False), ('TM', 1, False), ('TM', 2, False)), 3),\n",
       " ((('', 1, False), ('TM', 1, False), ('TM', 2, False)), 4),\n",
       " ((('', 1, False), ('TM', 1, True)), 2),\n",
       " ((('', 1, False), ('TM', 1, True)), 3),\n",
       " ((('', 1, False), ('TM', 2, False)), 3),\n",
       " ((('', 1, True),), 1),\n",
       " ((('', 2, False),), 1),\n",
       " ((('', 2, False),), 2),\n",
       " ((('', 2, False),), 3),\n",
       " ((('', 2, False),), 4),\n",
       " ((('', 2, False),), 5),\n",
       " ((('', 2, False),), 6),\n",
       " ((('', 2, False), ('', 2, True), ('TM', 3, False)), 6),\n",
       " ((('', 2, False), ('', 3, False)), 4),\n",
       " ((('', 2, False), ('', 3, False)), 5),\n",
       " ((('', 2, False), ('', 3, False)), 6),\n",
       " ((('', 2, False), ('T', 1, False)), 3),\n",
       " ((('', 2, False), ('T', 1, False)), 5),\n",
       " ((('', 2, False), ('T', 2, False)), 3),\n",
       " ((('', 2, False), ('T', 2, False)), 4),\n",
       " ((('', 2, False), ('T', 2, False)), 5),\n",
       " ((('', 2, False), ('TM', 1, False)), 2),\n",
       " ((('', 2, False), ('TM', 1, False)), 3),\n",
       " ((('', 2, False), ('TM', 1, False), ('TM', 1, True)), 4),\n",
       " ((('', 2, False), ('TM', 1, False), ('TM', 3, False)), 4),\n",
       " ((('', 2, False), ('TM', 1, True)), 2),\n",
       " ((('', 2, True),), 1),\n",
       " ((('', 2, True),), 2),\n",
       " ((('', 3, False),), 2),\n",
       " ((('', 3, False), ('T', 2, False)), 4),\n",
       " ((('', 3, False), ('T', 3, False)), 5),\n",
       " ((('', 3, True),), 2),\n",
       " ((('T', 1, False), ('TM', 1, False)), 2),\n",
       " ((('T', 2, False),), 1),\n",
       " ((('T', 2, False), ('T', 3, False)), 4),\n",
       " ((('T', 2, False), ('TM', 1, False)), 2),\n",
       " ((('T', 2, False), ('TM', 1, True)), 2),\n",
       " ((('T', 2, True),), 1),\n",
       " ((('T', 3, False),), 1),\n",
       " ((('T', 3, False),), 2),\n",
       " ((('T', 3, True),), 1),\n",
       " ((('T', 3, True),), 2),\n",
       " ((('TM', 1, False),), 1),\n",
       " ((('TM', 1, False),), 2),\n",
       " ((('TM', 1, False),), 3),\n",
       " ((('TM', 1, False), ('TM', 1, True)), 2),\n",
       " ((('TM', 1, False), ('TM', 2, False)), 2),\n",
       " ((('TM', 1, False), ('TM', 2, False)), 3),\n",
       " ((('TM', 1, False), ('TM', 3, False)), 2),\n",
       " ((('TM', 1, True),), 1),\n",
       " ((('TM', 2, False),), 1),\n",
       " ((('TM', 2, False),), 2),\n",
       " ((('TM', 2, False),), 3),\n",
       " ((('TM', 2, True),), 1),\n",
       " ((('TM', 3, False),), 2)}"
      ]
     },
     "execution_count": 286,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ment_set.keys() | gold_ment_set.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 287,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "((('', 2, False),), 4)\n",
      "12 mentions, 12 found, 11 correct.\n",
      "Precision: 0.92\n",
      "Recall:    0.92\n",
      "F1:        0.92\n",
      "FP ex.: ['ה מלחמת ה עצמאות']\n",
      "FN ex.: ['ה עולם ה שלישי']\n",
      "((('', 1, False), ('TM', 1, False)), 4)\n",
      "1 mentions, 0 found, 0 correct.\n",
      "Precision: -1\n",
      "Recall:    0.0\n",
      "F1:        0.0\n",
      "FP ex.: []\n",
      "FN ex.: ['באב אל - ואד']\n",
      "((('', 1, False),), 2)\n",
      "34 mentions, 37 found, 31 correct.\n",
      "Precision: 0.84\n",
      "Recall:    0.91\n",
      "F1:        0.87\n",
      "FP ex.: ['מחוז פאריס', 'קרן גון', 'סן סימון', 'נא מילוא', 'שלום עכשיו']\n",
      "FN ex.: ['מחוז פאריס', 'בית נבחרים', 'סנטר 1']\n",
      "((('', 1, False), ('', 2, False)), 4)\n",
      "7 mentions, 7 found, 4 correct.\n",
      "Precision: 0.57\n",
      "Recall:    0.57\n",
      "F1:        0.57\n",
      "FP ex.: ['הר - ה בית', 'שכונת שמואל ה נביא', 'שלום ה רב כהנא']\n",
      "FN ex.: ['הר - ה בית', 'שכונת שמואל ה נביא', 'פרס נובל ל שלום']\n",
      "((('', 1, False), ('', 2, False), ('', 3, False)), 10)\n",
      "1 mentions, 0 found, 0 correct.\n",
      "Precision: -1\n",
      "Recall:    0.0\n",
      "F1:        0.0\n",
      "FP ex.: []\n",
      "FN ex.: ['מרכז ה מידע ל זכויות ה אדם ב ה שטחים']\n",
      "((('T', 2, False), ('TM', 1, True)), 2)\n",
      "1 mentions, 1 found, 1 correct.\n",
      "Precision: 1.0\n",
      "Recall:    1.0\n",
      "F1:        1.0\n",
      "FP ex.: []\n",
      "FN ex.: []\n",
      "((('', 2, False),), 1)\n",
      "36 mentions, 38 found, 35 correct.\n",
      "Precision: 0.92\n",
      "Recall:    0.97\n",
      "F1:        0.95\n",
      "FP ex.: ['ירושלים', 'אושוויץ', 'מקורות']\n",
      "FN ex.: ['אושוויץ']\n",
      "((('TM', 3, False),), 2)\n",
      "0 mentions, 4 found, 0 correct.\n",
      "Precision: 0.0\n",
      "Recall:    -1\n",
      "F1:        0.0\n",
      "FP ex.: ['ה קטיף', 'ה קטיף', 'ה קטיף', 'ה קטיף']\n",
      "FN ex.: []\n",
      "((('T', 3, False),), 2)\n",
      "0 mentions, 1 found, 0 correct.\n",
      "Precision: 0.0\n",
      "Recall:    -1\n",
      "F1:        0.0\n",
      "FP ex.: ['ה ארה\"ב']\n",
      "FN ex.: []\n",
      "((('TM', 1, False), ('TM', 3, False)), 2)\n",
      "0 mentions, 1 found, 0 correct.\n",
      "Precision: 0.0\n",
      "Recall:    -1\n",
      "F1:        0.0\n",
      "FP ex.: ['איסט סייד']\n",
      "FN ex.: []\n",
      "((('', 2, True),), 2)\n",
      "1 mentions, 1 found, 0 correct.\n",
      "Precision: 0.0\n",
      "Recall:    0.0\n",
      "F1:        -1\n",
      "FP ex.: ['ה קב\"ה']\n",
      "FN ex.: ['ה קב\"ה']\n",
      "((('', 2, False), ('', 3, False)), 4)\n",
      "7 mentions, 6 found, 6 correct.\n",
      "Precision: 1.0\n",
      "Recall:    0.86\n",
      "F1:        0.92\n",
      "FP ex.: []\n",
      "FN ex.: ['ה עולם ה שלישי']\n",
      "((('', 2, False),), 2)\n",
      "12 mentions, 12 found, 5 correct.\n",
      "Precision: 0.42\n",
      "Recall:    0.42\n",
      "F1:        0.42\n",
      "FP ex.: ['ה שמרנים', 'ה ארץ', 'ה שמרנים', 'ה ארץ', 'ה תורה']\n",
      "FN ex.: ['ה גרמנים', 'ה מלחמה', 'ה משטרה', 'ה פנים', 'ה ארץ']\n",
      "((('TM', 1, False),), 3)\n",
      "2 mentions, 2 found, 2 correct.\n",
      "Precision: 1.0\n",
      "Recall:    1.0\n",
      "F1:        1.0\n",
      "FP ex.: []\n",
      "FN ex.: []\n",
      "((('', 2, False), ('T', 2, False)), 5)\n",
      "1 mentions, 1 found, 1 correct.\n",
      "Precision: 1.0\n",
      "Recall:    1.0\n",
      "F1:        1.0\n",
      "FP ex.: []\n",
      "FN ex.: []\n",
      "((('T', 3, False),), 1)\n",
      "3 mentions, 2 found, 2 correct.\n",
      "Precision: 1.0\n",
      "Recall:    0.67\n",
      "F1:        0.8\n",
      "FP ex.: []\n",
      "FN ex.: ['ארה\"ב']\n",
      "((('', 2, False), ('TM', 1, True)), 2)\n",
      "0 mentions, 1 found, 0 correct.\n",
      "Precision: 0.0\n",
      "Recall:    -1\n",
      "F1:        0.0\n",
      "FP ex.: ['הרי בראדלו']\n",
      "FN ex.: []\n",
      "((('', 1, False), ('TM', 2, False)), 3)\n",
      "2 mentions, 1 found, 1 correct.\n",
      "Precision: 1.0\n",
      "Recall:    0.5\n",
      "F1:        0.67\n",
      "FP ex.: []\n",
      "FN ex.: ['מנזר סן סימון']\n",
      "((('T', 2, False),), 1)\n",
      "11 mentions, 9 found, 9 correct.\n",
      "Precision: 1.0\n",
      "Recall:    0.82\n",
      "F1:        0.9\n",
      "FP ex.: []\n",
      "FN ex.: ['קדיש', 'לין']\n",
      "((('TM', 2, True),), 1)\n",
      "2 mentions, 1 found, 1 correct.\n",
      "Precision: 1.0\n",
      "Recall:    0.5\n",
      "F1:        0.67\n",
      "FP ex.: []\n",
      "FN ex.: ['קריסטול']\n",
      "((('', 2, False), ('TM', 1, False), ('TM', 3, False)), 4)\n",
      "1 mentions, 0 found, 0 correct.\n",
      "Precision: -1\n",
      "Recall:    0.0\n",
      "F1:        0.0\n",
      "FP ex.: []\n",
      "FN ex.: ['איסט סייד ה תחתית']\n",
      "((('', 1, False), ('', 2, False)), 2)\n",
      "6 mentions, 4 found, 4 correct.\n",
      "Precision: 1.0\n",
      "Recall:    0.67\n",
      "F1:        0.8\n",
      "FP ex.: []\n",
      "FN ex.: ['מלחמת וייטנאם', 'ניו גרסי']\n",
      "((('', 1, False), ('T', 2, False), ('TM', 1, False)), 4)\n",
      "0 mentions, 1 found, 0 correct.\n",
      "Precision: 0.0\n",
      "Recall:    -1\n",
      "F1:        0.0\n",
      "FP ex.: ['ארגון נפגעי ה משכנתאות']\n",
      "FN ex.: []\n",
      "((('TM', 1, False), ('TM', 2, False)), 3)\n",
      "1 mentions, 0 found, 0 correct.\n",
      "Precision: -1\n",
      "Recall:    0.0\n",
      "F1:        0.0\n",
      "FP ex.: []\n",
      "FN ex.: ['תיאוריית ה קוונטים']\n",
      "((('', 1, False), ('TM', 1, False), ('TM', 1, True)), 5)\n",
      "0 mentions, 1 found, 0 correct.\n",
      "Precision: 0.0\n",
      "Recall:    -1\n",
      "F1:        0.0\n",
      "FP ex.: ['אייפא\"ק טום דאיין את הארקין']\n",
      "FN ex.: []\n",
      "((('', 1, False), ('TM', 1, True)), 3)\n",
      "2 mentions, 2 found, 1 correct.\n",
      "Precision: 0.5\n",
      "Recall:    0.5\n",
      "F1:        0.5\n",
      "FP ex.: ['אמריקן אקונומיק ריוויו']\n",
      "FN ex.: ['אמריקן אקונומיק ריוויו']\n",
      "((('TM', 2, False),), 1)\n",
      "21 mentions, 19 found, 19 correct.\n",
      "Precision: 1.0\n",
      "Recall:    0.9\n",
      "F1:        0.95\n",
      "FP ex.: []\n",
      "FN ex.: ['ליון', 'אלוקים']\n",
      "((('', 1, False), ('T', 2, False)), 5)\n",
      "2 mentions, 0 found, 0 correct.\n",
      "Precision: -1\n",
      "Recall:    0.0\n",
      "F1:        0.0\n",
      "FP ex.: []\n",
      "FN ex.: ['ה קרב על סן סימון', 'ה קרב על סן סימון']\n",
      "((('', 1, False), ('T', 1, False), ('TM', 1, False)), 5)\n",
      "0 mentions, 1 found, 0 correct.\n",
      "Precision: 0.0\n",
      "Recall:    -1\n",
      "F1:        0.0\n",
      "FP ex.: ['כביש באב אל - ואד']\n",
      "FN ex.: []\n",
      "((('', 2, False),), 5)\n",
      "1 mentions, 1 found, 0 correct.\n",
      "Precision: 0.0\n",
      "Recall:    0.0\n",
      "F1:        -1\n",
      "FP ex.: ['ישיבת ה רעיון ה יהודי']\n",
      "FN ex.: ['ישיבת ה רעיון ה יהודי']\n",
      "((('', 1, False), ('', 3, False)), 4)\n",
      "1 mentions, 0 found, 0 correct.\n",
      "Precision: -1\n",
      "Recall:    0.0\n",
      "F1:        0.0\n",
      "FP ex.: []\n",
      "FN ex.: ['ה קרב על ירושלים']\n",
      "((('', 2, False), ('T', 2, False)), 3)\n",
      "3 mentions, 2 found, 2 correct.\n",
      "Precision: 1.0\n",
      "Recall:    0.67\n",
      "F1:        0.8\n",
      "FP ex.: []\n",
      "FN ex.: ['תנועת ה מושבים']\n",
      "((('', 1, False), ('TM', 1, False), ('TM', 2, False)), 3)\n",
      "2 mentions, 2 found, 1 correct.\n",
      "Precision: 0.5\n",
      "Recall:    0.5\n",
      "F1:        0.5\n",
      "FP ex.: ['נווה - מונסון']\n",
      "FN ex.: ['נווה - מונסון']\n",
      "((('TM', 1, True),), 1)\n",
      "14 mentions, 13 found, 11 correct.\n",
      "Precision: 0.85\n",
      "Recall:    0.79\n",
      "F1:        0.81\n",
      "FP ex.: ['פראפראז', 'ירסקו']\n",
      "FN ex.: ['אייפא\"ק', 'פראפראז', 'נבראסקה']\n",
      "((('', 1, False),), 1)\n",
      "60 mentions, 54 found, 46 correct.\n",
      "Precision: 0.85\n",
      "Recall:    0.77\n",
      "F1:        0.81\n",
      "FP ex.: ['הראל', 'ירושלים', 'יונה', 'גורג', 'גרסי']\n",
      "FN ex.: ['ה', 'כך', 'וייטנאם', 'פיו', 'גורג']\n",
      "((('', 1, False), ('', 2, False)), 5)\n",
      "4 mentions, 4 found, 4 correct.\n",
      "Precision: 1.0\n",
      "Recall:    1.0\n",
      "F1:        1.0\n",
      "FP ex.: []\n",
      "FN ex.: []\n",
      "((('', 1, False), ('', 2, False), ('TM', 1, False)), 3)\n",
      "0 mentions, 1 found, 0 correct.\n",
      "Precision: 0.0\n",
      "Recall:    -1\n",
      "F1:        0.0\n",
      "FP ex.: ['קרן פורד שמרנים']\n",
      "FN ex.: []\n",
      "((('', 1, False), ('', 2, False), ('', 3, False)), 6)\n",
      "1 mentions, 1 found, 1 correct.\n",
      "Precision: 1.0\n",
      "Recall:    1.0\n",
      "F1:        1.0\n",
      "FP ex.: []\n",
      "FN ex.: []\n",
      "((('', 2, False),), 6)\n",
      "1 mentions, 0 found, 0 correct.\n",
      "Precision: -1\n",
      "Recall:    0.0\n",
      "F1:        0.0\n",
      "FP ex.: []\n",
      "FN ex.: ['ה ליגה ל זכויות ה אדם']\n",
      "((('', 1, False), ('TM', 1, False), ('TM', 1, True)), 3)\n",
      "2 mentions, 2 found, 0 correct.\n",
      "Precision: 0.0\n",
      "Recall:    0.0\n",
      "F1:        -1\n",
      "FP ex.: ['גנאדאס דל ואיה', 'סמית - ריצרדסון']\n",
      "FN ex.: ['גנאדאס דל ואיה', 'סמית - ריצרדסון']\n",
      "((('', 1, False), ('T', 2, False), ('TM', 2, False)), 4)\n",
      "1 mentions, 0 found, 0 correct.\n",
      "Precision: -1\n",
      "Recall:    0.0\n",
      "F1:        0.0\n",
      "FP ex.: []\n",
      "FN ex.: ['נאום אימפריית ה רשע']\n",
      "((('', 1, False), ('T', 1, False)), 2)\n",
      "2 mentions, 2 found, 2 correct.\n",
      "Precision: 1.0\n",
      "Recall:    1.0\n",
      "F1:        1.0\n",
      "FP ex.: []\n",
      "FN ex.: []\n",
      "((('', 2, True),), 1)\n",
      "1 mentions, 2 found, 1 correct.\n",
      "Precision: 0.5\n",
      "Recall:    1.0\n",
      "F1:        0.67\n",
      "FP ex.: ['שיקאגו']\n",
      "FN ex.: []\n",
      "((('', 1, False), ('', 2, False)), 6)\n",
      "1 mentions, 0 found, 0 correct.\n",
      "Precision: -1\n",
      "Recall:    0.0\n",
      "F1:        0.0\n",
      "FP ex.: []\n",
      "FN ex.: ['ה התאחדות ל עולם טוב יותר']\n",
      "((('', 2, False), ('', 3, False)), 5)\n",
      "1 mentions, 1 found, 1 correct.\n",
      "Precision: 1.0\n",
      "Recall:    1.0\n",
      "F1:        1.0\n",
      "FP ex.: []\n",
      "FN ex.: []\n",
      "((('', 3, False), ('T', 2, False)), 4)\n",
      "3 mentions, 0 found, 0 correct.\n",
      "Precision: -1\n",
      "Recall:    0.0\n",
      "F1:        0.0\n",
      "FP ex.: []\n",
      "FN ex.: ['ה בית ה לבן', 'ה בית ה לבן', 'ה מלחמה ה קרה']\n",
      "((('', 2, False),), 3)\n",
      "5 mentions, 4 found, 3 correct.\n",
      "Precision: 0.75\n",
      "Recall:    0.6\n",
      "F1:        0.67\n",
      "FP ex.: ['מדינות ה ברית']\n",
      "FN ex.: ['ירושלים ה מערבית', 'מלחמת ה עצמאות']\n",
      "((('', 2, False), ('TM', 1, False), ('TM', 1, True)), 4)\n",
      "1 mentions, 0 found, 0 correct.\n",
      "Precision: -1\n",
      "Recall:    0.0\n",
      "F1:        0.0\n",
      "FP ex.: []\n",
      "FN ex.: ['לינד ו הרי בראדלו']\n",
      "((('', 2, False), ('T', 2, False)), 4)\n",
      "4 mentions, 0 found, 0 correct.\n",
      "Precision: -1\n",
      "Recall:    0.0\n",
      "F1:        0.0\n",
      "FP ex.: []\n",
      "FN ex.: ['ה בית ה לבן', 'ה מלחמה ה קרה', 'ה בית ה לבן', 'ה מצביע ה אמריקאי']\n",
      "((('TM', 1, False), ('TM', 1, True)), 2)\n",
      "4 mentions, 4 found, 4 correct.\n",
      "Precision: 1.0\n",
      "Recall:    1.0\n",
      "F1:        1.0\n",
      "FP ex.: []\n",
      "FN ex.: []\n",
      "((('', 1, True),), 1)\n",
      "13 mentions, 14 found, 10 correct.\n",
      "Precision: 0.71\n",
      "Recall:    0.77\n",
      "F1:        0.74\n",
      "FP ex.: ['מסצוסטס', 'גרוסבורד', 'מסצוסטס', 'מסצוסטס']\n",
      "FN ex.: ['מסצוסטס', 'מסצוסטס', 'מסצוסטס']\n",
      "((('', 1, False), ('', 2, False)), 3)\n",
      "14 mentions, 16 found, 12 correct.\n",
      "Precision: 0.75\n",
      "Recall:    0.86\n",
      "F1:        0.8\n",
      "FP ex.: ['רחוב גבעת שאול', 'גורג ה מסכן', 'רצח ה נזירות', 'ה שירות פועל']\n",
      "FN ex.: ['רחוב גבעת שאול', 'ניו - יורק']\n",
      "((('', 2, False), ('', 3, False)), 6)\n",
      "2 mentions, 2 found, 2 correct.\n",
      "Precision: 1.0\n",
      "Recall:    1.0\n",
      "F1:        1.0\n",
      "FP ex.: []\n",
      "FN ex.: []\n",
      "((('', 1, False), ('', 2, False), ('T', 2, False)), 5)\n",
      "1 mentions, 1 found, 1 correct.\n",
      "Precision: 1.0\n",
      "Recall:    1.0\n",
      "F1:        1.0\n",
      "FP ex.: []\n",
      "FN ex.: []\n",
      "((('', 1, False), ('TM', 1, True)), 2)\n",
      "6 mentions, 6 found, 4 correct.\n",
      "Precision: 0.67\n",
      "Recall:    0.67\n",
      "F1:        0.67\n",
      "FP ex.: ['מחוז זירונד', 'שרה סקאיף']\n",
      "FN ex.: ['שרה סקאיף', 'מחוז זירונד']\n",
      "((('', 2, False), ('TM', 1, False)), 3)\n",
      "1 mentions, 0 found, 0 correct.\n",
      "Precision: -1\n",
      "Recall:    0.0\n",
      "F1:        0.0\n",
      "FP ex.: []\n",
      "FN ex.: ['מלאך ה מוות']\n",
      "((('', 1, False), ('T', 2, False)), 2)\n",
      "5 mentions, 4 found, 4 correct.\n",
      "Precision: 1.0\n",
      "Recall:    0.8\n",
      "F1:        0.89\n",
      "FP ex.: []\n",
      "FN ex.: ['עין כרם']\n",
      "((('T', 1, False), ('TM', 1, False)), 2)\n",
      "0 mentions, 1 found, 0 correct.\n",
      "Precision: 0.0\n",
      "Recall:    -1\n",
      "F1:        0.0\n",
      "FP ex.: ['שכונת קטמון']\n",
      "FN ex.: []\n",
      "((('T', 3, True),), 1)\n",
      "1 mentions, 0 found, 0 correct.\n",
      "Precision: -1\n",
      "Recall:    0.0\n",
      "F1:        0.0\n",
      "FP ex.: []\n",
      "FN ex.: ['מסצוסטס']\n",
      "((('', 1, False), ('', 1, True)), 2)\n",
      "5 mentions, 5 found, 5 correct.\n",
      "Precision: 1.0\n",
      "Recall:    1.0\n",
      "F1:        1.0\n",
      "FP ex.: []\n",
      "FN ex.: []\n",
      "((('T', 2, False), ('TM', 1, False)), 2)\n",
      "2 mentions, 2 found, 2 correct.\n",
      "Precision: 1.0\n",
      "Recall:    1.0\n",
      "F1:        1.0\n",
      "FP ex.: []\n",
      "FN ex.: []\n",
      "((('', 1, False), ('TM', 1, False)), 2)\n",
      "34 mentions, 33 found, 32 correct.\n",
      "Precision: 0.97\n",
      "Recall:    0.94\n",
      "F1:        0.96\n",
      "FP ex.: ['דה מוין']\n",
      "FN ex.: ['דה מוין', 'מכון הדסון']\n",
      "((('TM', 2, False),), 3)\n",
      "1 mentions, 1 found, 0 correct.\n",
      "Precision: 0.0\n",
      "Recall:    0.0\n",
      "F1:        -1\n",
      "FP ex.: ['קטיף ה הדרים']\n",
      "FN ex.: ['תיאוריית ה יחסות']\n",
      "((('', 1, False), ('', 2, False), ('T', 2, False), ('TM', 1, False)), 8)\n",
      "1 mentions, 0 found, 0 correct.\n",
      "Precision: -1\n",
      "Recall:    0.0\n",
      "F1:        0.0\n",
      "FP ex.: []\n",
      "FN ex.: ['ארגון נפגעי ה משכנתאות ו חסרי ה דיור']\n",
      "((('', 1, False), ('', 2, False), ('TM', 1, False), ('TM', 2, False)), 14)\n",
      "1 mentions, 0 found, 0 correct.\n",
      "Precision: -1\n",
      "Recall:    0.0\n",
      "F1:        0.0\n",
      "FP ex.: []\n",
      "FN ex.: ['ה ברוקרים של ה רעיונות : צוותות חשיבה ו עלייתה של עלית מדיניות חדשה']\n",
      "((('', 2, False), ('', 2, True), ('TM', 3, False)), 6)\n",
      "1 mentions, 0 found, 0 correct.\n",
      "Precision: -1\n",
      "Recall:    0.0\n",
      "F1:        0.0\n",
      "FP ex.: []\n",
      "FN ex.: ['ה קונסוליה ה ישראלית ב שיקאגו']\n",
      "((('', 1, False), ('T', 3, False), ('TM', 1, False)), 6)\n",
      "1 mentions, 0 found, 0 correct.\n",
      "Precision: -1\n",
      "Recall:    0.0\n",
      "F1:        0.0\n",
      "FP ex.: []\n",
      "FN ex.: ['ה קרב על מנזר סן סימון']\n",
      "((('', 1, False),), 3)\n",
      "4 mentions, 3 found, 3 correct.\n",
      "Precision: 1.0\n",
      "Recall:    0.75\n",
      "F1:        0.86\n",
      "FP ex.: []\n",
      "FN ex.: ['בית מספר 3']\n",
      "((('', 1, False), ('', 2, False)), 7)\n",
      "1 mentions, 1 found, 1 correct.\n",
      "Precision: 1.0\n",
      "Recall:    1.0\n",
      "F1:        1.0\n",
      "FP ex.: []\n",
      "FN ex.: []\n",
      "((('', 2, False), ('T', 1, False)), 3)\n",
      "0 mentions, 1 found, 0 correct.\n",
      "Precision: 0.0\n",
      "Recall:    -1\n",
      "F1:        0.0\n",
      "FP ex.: ['ישיבת ה רעיון']\n",
      "FN ex.: []\n",
      "((('T', 2, True),), 1)\n",
      "1 mentions, 1 found, 1 correct.\n",
      "Precision: 1.0\n",
      "Recall:    1.0\n",
      "F1:        1.0\n",
      "FP ex.: []\n",
      "FN ex.: []\n",
      "((('', 3, False),), 2)\n",
      "10 mentions, 10 found, 8 correct.\n",
      "Precision: 0.8\n",
      "Recall:    0.8\n",
      "F1:        0.8\n",
      "FP ex.: ['ה שטחים', 'ה ארץ']\n",
      "FN ex.: ['ה ארץ', 'ה ארץ']\n",
      "((('TM', 1, False),), 1)\n",
      "65 mentions, 54 found, 48 correct.\n",
      "Precision: 0.89\n",
      "Recall:    0.74\n",
      "F1:        0.81\n",
      "FP ex.: ['קטמון', 'לינד', 'גאנט', 'אולין', 'קטמון']\n",
      "FN ex.: ['רוקפלר', 'הארקין', 'מפם', 'טוקי', 'סילבר']\n",
      "((('T', 2, False), ('T', 3, False)), 4)\n",
      "1 mentions, 0 found, 0 correct.\n",
      "Precision: -1\n",
      "Recall:    0.0\n",
      "F1:        0.0\n",
      "FP ex.: []\n",
      "FN ex.: ['ה מלחמה ה קרה']\n",
      "((('', 3, False), ('T', 3, False)), 5)\n",
      "1 mentions, 0 found, 0 correct.\n",
      "Precision: -1\n",
      "Recall:    0.0\n",
      "F1:        0.0\n",
      "FP ex.: []\n",
      "FN ex.: ['ה עבודה ו ה רווחה']\n",
      "((('', 1, False), ('', 2, False), ('TM', 2, False)), 6)\n",
      "1 mentions, 0 found, 0 correct.\n",
      "Precision: -1\n",
      "Recall:    0.0\n",
      "F1:        0.0\n",
      "FP ex.: []\n",
      "FN ex.: ['ה שדולה ה פרו - ישראלית']\n",
      "((('TM', 1, False),), 2)\n",
      "18 mentions, 18 found, 17 correct.\n",
      "Precision: 0.94\n",
      "Recall:    0.94\n",
      "F1:        0.94\n",
      "FP ex.: ['ל אקספרס']\n",
      "FN ex.: ['טום דאיין']\n",
      "((('', 2, False), ('TM', 1, False)), 2)\n",
      "7 mentions, 1 found, 1 correct.\n",
      "Precision: 1.0\n",
      "Recall:    0.14\n",
      "F1:        0.25\n",
      "FP ex.: []\n",
      "FN ex.: ['חברה קדישא', 'חברה קדישא', 'קרן פורד', 'חברה קדישא', 'לוס אוחוס']\n",
      "((('', 1, False), ('TM', 1, False), ('TM', 2, False)), 4)\n",
      "1 mentions, 0 found, 0 correct.\n",
      "Precision: -1\n",
      "Recall:    0.0\n",
      "F1:        0.0\n",
      "FP ex.: []\n",
      "FN ex.: ['מענק \" גאונות \"']\n",
      "((('', 1, False), ('T', 2, False)), 3)\n",
      "4 mentions, 1 found, 1 correct.\n",
      "Precision: 1.0\n",
      "Recall:    0.25\n",
      "F1:        0.4\n",
      "FP ex.: []\n",
      "FN ex.: ['תנועת ה מושבים', 'תנועת ה מושבים', 'תנועת ה מושבים']\n",
      "((('', 1, False),), 4)\n",
      "2 mentions, 1 found, 1 correct.\n",
      "Precision: 1.0\n",
      "Recall:    0.5\n",
      "F1:        0.67\n",
      "FP ex.: []\n",
      "FN ex.: ['חטיבת \" הראל \"']\n",
      "((('', 2, False), ('T', 1, False)), 5)\n",
      "1 mentions, 0 found, 0 correct.\n",
      "Precision: -1\n",
      "Recall:    0.0\n",
      "F1:        0.0\n",
      "FP ex.: []\n",
      "FN ex.: ['ישיבת ה רעיון ה יהודי']\n",
      "((('T', 3, True),), 2)\n",
      "0 mentions, 1 found, 0 correct.\n",
      "Precision: 0.0\n",
      "Recall:    -1\n",
      "F1:        0.0\n",
      "FP ex.: ['ה מסצוסטס']\n",
      "FN ex.: []\n",
      "((('TM', 1, False), ('TM', 2, False)), 2)\n",
      "3 mentions, 3 found, 3 correct.\n",
      "Precision: 1.0\n",
      "Recall:    1.0\n",
      "F1:        1.0\n",
      "FP ex.: []\n",
      "FN ex.: []\n",
      "((('', 1, False), ('', 2, False), ('T', 1, False)), 4)\n",
      "1 mentions, 0 found, 0 correct.\n",
      "Precision: -1\n",
      "Recall:    0.0\n",
      "F1:        0.0\n",
      "FP ex.: []\n",
      "FN ex.: ['צדקה מתחילה מ בית']\n",
      "((('', 3, True),), 2)\n",
      "1 mentions, 0 found, 0 correct.\n",
      "Precision: -1\n",
      "Recall:    0.0\n",
      "F1:        0.0\n",
      "FP ex.: []\n",
      "FN ex.: ['ה כנסת']\n",
      "((('', 1, False), ('TM', 1, False), ('TM', 1, True)), 4)\n",
      "2 mentions, 2 found, 2 correct.\n",
      "Precision: 1.0\n",
      "Recall:    1.0\n",
      "F1:        1.0\n",
      "FP ex.: []\n",
      "FN ex.: []\n",
      "((('', 1, False), ('TM', 1, False)), 3)\n",
      "5 mentions, 6 found, 4 correct.\n",
      "Precision: 0.67\n",
      "Recall:    0.8\n",
      "F1:        0.73\n",
      "FP ex.: ['מנזר סן סימון', 'מכון ה דסון']\n",
      "FN ex.: ['מנזר סן סימון']\n",
      "((('TM', 2, False),), 2)\n",
      "3 mentions, 0 found, 0 correct.\n",
      "Precision: -1\n",
      "Recall:    0.0\n",
      "F1:        0.0\n",
      "FP ex.: []\n",
      "FN ex.: ['ה פלמ\"ח', 'ה גסטאפו', 'ה גסטאפו']\n"
     ]
    }
   ],
   "source": [
    "for k in ment_set.keys() | gold_ment_set.keys():\n",
    "    print(k)\n",
    "    nem.evaluate_mentions(gold_ment_set[k], ment_set[k])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Evaluate all"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 167,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>gold_name</th>\n",
       "      <th>unit</th>\n",
       "      <th>arch</th>\n",
       "      <th>w_embed</th>\n",
       "      <th>seed_num</th>\n",
       "      <th>p_m</th>\n",
       "      <th>r_m</th>\n",
       "      <th>f_m</th>\n",
       "      <th>pred_set</th>\n",
       "      <th>acc</th>\n",
       "      <th>...</th>\n",
       "      <th>seg</th>\n",
       "      <th>model_file_name</th>\n",
       "      <th>dset_file_name</th>\n",
       "      <th>relevant_score</th>\n",
       "      <th>input_unit</th>\n",
       "      <th>embed_unit</th>\n",
       "      <th>embed_type</th>\n",
       "      <th>cm</th>\n",
       "      <th>pred_set_sub</th>\n",
       "      <th>pred_set_main</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>morph_dev_gold</td>\n",
       "      <td>morph</td>\n",
       "      <td>char_cnn</td>\n",
       "      <td>ft_yap</td>\n",
       "      <td>44_seed</td>\n",
       "      <td>0.859155</td>\n",
       "      <td>0.733467</td>\n",
       "      <td>0.791351</td>\n",
       "      <td>dev_gold</td>\n",
       "      <td>0.9667</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>morph.char_cnn.ft_yap.44_seed.83.model</td>\n",
       "      <td>morph.char_cnn.ft_yap.44_seed.dset</td>\n",
       "      <td>0.7905</td>\n",
       "      <td>morph</td>\n",
       "      <td>morph</td>\n",
       "      <td>ft</td>\n",
       "      <td>Match</td>\n",
       "      <td>gold</td>\n",
       "      <td>dev</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>morph_dev_yap</td>\n",
       "      <td>morph</td>\n",
       "      <td>char_cnn</td>\n",
       "      <td>ft_yap</td>\n",
       "      <td>44_seed</td>\n",
       "      <td>0.780193</td>\n",
       "      <td>0.647295</td>\n",
       "      <td>0.707558</td>\n",
       "      <td>dev_yap</td>\n",
       "      <td>0.9667</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>morph.char_cnn.ft_yap.44_seed.83.model</td>\n",
       "      <td>morph.char_cnn.ft_yap.44_seed.dset</td>\n",
       "      <td>0.7905</td>\n",
       "      <td>morph</td>\n",
       "      <td>morph</td>\n",
       "      <td>ft</td>\n",
       "      <td>Match</td>\n",
       "      <td>yap</td>\n",
       "      <td>dev</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>morph_test_gold</td>\n",
       "      <td>morph</td>\n",
       "      <td>char_cnn</td>\n",
       "      <td>ft_yap</td>\n",
       "      <td>44_seed</td>\n",
       "      <td>0.804850</td>\n",
       "      <td>0.747854</td>\n",
       "      <td>0.775306</td>\n",
       "      <td>test_gold</td>\n",
       "      <td>0.9667</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>morph.char_cnn.ft_yap.44_seed.83.model</td>\n",
       "      <td>morph.char_cnn.ft_yap.44_seed.dset</td>\n",
       "      <td>0.7905</td>\n",
       "      <td>morph</td>\n",
       "      <td>morph</td>\n",
       "      <td>ft</td>\n",
       "      <td>Match</td>\n",
       "      <td>gold</td>\n",
       "      <td>test</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>morph_dev_gold</td>\n",
       "      <td>morph</td>\n",
       "      <td>char_cnn</td>\n",
       "      <td>ft_oov_tok</td>\n",
       "      <td>44_seed</td>\n",
       "      <td>0.843612</td>\n",
       "      <td>0.767535</td>\n",
       "      <td>0.803778</td>\n",
       "      <td>dev_gold</td>\n",
       "      <td>0.9690</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>morph.char_cnn.ft_oov_tok.44_seed.183.model</td>\n",
       "      <td>morph.char_cnn.ft_oov_tok.44_seed.dset</td>\n",
       "      <td>0.8038</td>\n",
       "      <td>morph</td>\n",
       "      <td>token</td>\n",
       "      <td>ft_oov</td>\n",
       "      <td>Clash</td>\n",
       "      <td>gold</td>\n",
       "      <td>dev</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>morph_test_yap</td>\n",
       "      <td>morph</td>\n",
       "      <td>char_cnn</td>\n",
       "      <td>ft_yap</td>\n",
       "      <td>44_seed</td>\n",
       "      <td>0.721411</td>\n",
       "      <td>0.636266</td>\n",
       "      <td>0.676169</td>\n",
       "      <td>test_yap</td>\n",
       "      <td>0.9667</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>morph.char_cnn.ft_yap.44_seed.83.model</td>\n",
       "      <td>morph.char_cnn.ft_yap.44_seed.dset</td>\n",
       "      <td>0.7905</td>\n",
       "      <td>morph</td>\n",
       "      <td>morph</td>\n",
       "      <td>ft</td>\n",
       "      <td>Match</td>\n",
       "      <td>yap</td>\n",
       "      <td>test</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 56 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         gold_name   unit      arch     w_embed seed_num       p_m       r_m  \\\n",
       "0   morph_dev_gold  morph  char_cnn      ft_yap  44_seed  0.859155  0.733467   \n",
       "1    morph_dev_yap  morph  char_cnn      ft_yap  44_seed  0.780193  0.647295   \n",
       "2  morph_test_gold  morph  char_cnn      ft_yap  44_seed  0.804850  0.747854   \n",
       "3   morph_dev_gold  morph  char_cnn  ft_oov_tok  44_seed  0.843612  0.767535   \n",
       "4   morph_test_yap  morph  char_cnn      ft_yap  44_seed  0.721411  0.636266   \n",
       "\n",
       "        f_m   pred_set     acc  ...  seg  \\\n",
       "0  0.791351   dev_gold  0.9667  ...  NaN   \n",
       "1  0.707558    dev_yap  0.9667  ...  NaN   \n",
       "2  0.775306  test_gold  0.9667  ...  NaN   \n",
       "3  0.803778   dev_gold  0.9690  ...  NaN   \n",
       "4  0.676169   test_yap  0.9667  ...  NaN   \n",
       "\n",
       "                               model_file_name  \\\n",
       "0       morph.char_cnn.ft_yap.44_seed.83.model   \n",
       "1       morph.char_cnn.ft_yap.44_seed.83.model   \n",
       "2       morph.char_cnn.ft_yap.44_seed.83.model   \n",
       "3  morph.char_cnn.ft_oov_tok.44_seed.183.model   \n",
       "4       morph.char_cnn.ft_yap.44_seed.83.model   \n",
       "\n",
       "                           dset_file_name relevant_score input_unit  \\\n",
       "0      morph.char_cnn.ft_yap.44_seed.dset         0.7905      morph   \n",
       "1      morph.char_cnn.ft_yap.44_seed.dset         0.7905      morph   \n",
       "2      morph.char_cnn.ft_yap.44_seed.dset         0.7905      morph   \n",
       "3  morph.char_cnn.ft_oov_tok.44_seed.dset         0.8038      morph   \n",
       "4      morph.char_cnn.ft_yap.44_seed.dset         0.7905      morph   \n",
       "\n",
       "  embed_unit embed_type     cm pred_set_sub pred_set_main  \n",
       "0      morph         ft  Match         gold           dev  \n",
       "1      morph         ft  Match          yap           dev  \n",
       "2      morph         ft  Match         gold          test  \n",
       "3      token     ft_oov  Clash         gold           dev  \n",
       "4      morph         ft  Match          yap          test  \n",
       "\n",
       "[5 rows x 56 columns]"
      ]
     },
     "execution_count": 167,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mev = pd.read_pickle('final_setup/mev2.pkl')\n",
    "mev.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 189,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>form</th>\n",
       "      <th>lemma</th>\n",
       "      <th>upostag</th>\n",
       "      <th>xpostag</th>\n",
       "      <th>feats</th>\n",
       "      <th>token_id</th>\n",
       "      <th>sent_id</th>\n",
       "      <th>token_str</th>\n",
       "      <th>global_sent_id</th>\n",
       "      <th>...</th>\n",
       "      <th>biose_layer1</th>\n",
       "      <th>biose_layer2</th>\n",
       "      <th>biose_layer3</th>\n",
       "      <th>token_unk</th>\n",
       "      <th>morph_unk</th>\n",
       "      <th>lemma_unk</th>\n",
       "      <th>unk_type</th>\n",
       "      <th>has_ner</th>\n",
       "      <th>morpheme_count</th>\n",
       "      <th>token_id_rep</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>16823</th>\n",
       "      <td>5</td>\n",
       "      <td>אך</td>\n",
       "      <td>אך</td>\n",
       "      <td>CC</td>\n",
       "      <td>CC</td>\n",
       "      <td>_</td>\n",
       "      <td>4</td>\n",
       "      <td>706</td>\n",
       "      <td>אך</td>\n",
       "      <td>6153</td>\n",
       "      <td>...</td>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td></td>\n",
       "      <td>False</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16824</th>\n",
       "      <td>6</td>\n",
       "      <td>לא</td>\n",
       "      <td>לא</td>\n",
       "      <td>RB</td>\n",
       "      <td>RB</td>\n",
       "      <td>_</td>\n",
       "      <td>5</td>\n",
       "      <td>706</td>\n",
       "      <td>לא</td>\n",
       "      <td>6153</td>\n",
       "      <td>...</td>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td></td>\n",
       "      <td>False</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16825</th>\n",
       "      <td>7</td>\n",
       "      <td>ה</td>\n",
       "      <td>ה</td>\n",
       "      <td>DEF</td>\n",
       "      <td>DEF</td>\n",
       "      <td>_</td>\n",
       "      <td>6</td>\n",
       "      <td>706</td>\n",
       "      <td>התרופה</td>\n",
       "      <td>6153</td>\n",
       "      <td>...</td>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td></td>\n",
       "      <td>False</td>\n",
       "      <td>2</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16826</th>\n",
       "      <td>8</td>\n",
       "      <td>תרופה</td>\n",
       "      <td>תרופה</td>\n",
       "      <td>NN</td>\n",
       "      <td>NN</td>\n",
       "      <td>gen=F|num=S</td>\n",
       "      <td>6</td>\n",
       "      <td>706</td>\n",
       "      <td>התרופה</td>\n",
       "      <td>6153</td>\n",
       "      <td>...</td>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td></td>\n",
       "      <td>False</td>\n",
       "      <td>2</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16827</th>\n",
       "      <td>9</td>\n",
       "      <td>.</td>\n",
       "      <td>_</td>\n",
       "      <td>yyDOT</td>\n",
       "      <td>yyDOT</td>\n",
       "      <td>_</td>\n",
       "      <td>7</td>\n",
       "      <td>706</td>\n",
       "      <td>.</td>\n",
       "      <td>6153</td>\n",
       "      <td>...</td>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td></td>\n",
       "      <td>False</td>\n",
       "      <td>1</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 34 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       id   form  lemma upostag xpostag        feats  token_id  sent_id  \\\n",
       "16823   5     אך     אך      CC      CC            _         4      706   \n",
       "16824   6     לא     לא      RB      RB            _         5      706   \n",
       "16825   7      ה      ה     DEF     DEF            _         6      706   \n",
       "16826   8  תרופה  תרופה      NN      NN  gen=F|num=S         6      706   \n",
       "16827   9      .      _   yyDOT   yyDOT            _         7      706   \n",
       "\n",
       "      token_str  global_sent_id  ...  biose_layer1 biose_layer2 biose_layer3  \\\n",
       "16823        אך            6153  ...             O            O            O   \n",
       "16824        לא            6153  ...             O            O            O   \n",
       "16825    התרופה            6153  ...             O            O            O   \n",
       "16826    התרופה            6153  ...             O            O            O   \n",
       "16827         .            6153  ...             O            O            O   \n",
       "\n",
       "       token_unk morph_unk  lemma_unk unk_type has_ner morpheme_count  \\\n",
       "16823      False     False      False            False              1   \n",
       "16824      False     False      False            False              1   \n",
       "16825      False     False      False            False              2   \n",
       "16826      False     False      False            False              2   \n",
       "16827      False     False      False            False              1   \n",
       "\n",
       "      token_id_rep  \n",
       "16823            4  \n",
       "16824            5  \n",
       "16825            6  \n",
       "16826            6  \n",
       "16827            7  \n",
       "\n",
       "[5 rows x 34 columns]"
      ]
     },
     "execution_count": 189,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dev_gold = spdf[spdf.set=='dev']\n",
    "dev_gold = dev_gold.merge(dev_unk, how='left')\n",
    "test_gold = spdf[spdf.set=='test']\n",
    "test_gold = test_gold.merge(test_unk, how='left')\n",
    "test_sent_id_map = (test_gold.groupby('sent_id').size()\n",
    "                    .reset_index().drop(0, axis=1).reset_index()\n",
    "                    .assign(index=lambda x: x+1).set_index('sent_id')['index'])\n",
    "\n",
    "test_gold['sent_id'] = test_gold.sent_id.map(test_sent_id_map)\n",
    "test_unk['sent_id'] = test_unk.sent_id.map(test_sent_id_map)\n",
    "test_gold.tail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 190,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sent_id</th>\n",
       "      <th>token_id</th>\n",
       "      <th>token_str</th>\n",
       "      <th>token_unk</th>\n",
       "      <th>morph_unk</th>\n",
       "      <th>lemma_unk</th>\n",
       "      <th>unk_type</th>\n",
       "      <th>has_ner</th>\n",
       "      <th>morpheme_count</th>\n",
       "      <th>token_id_rep</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>12614</th>\n",
       "      <td>706</td>\n",
       "      <td>3</td>\n",
       "      <td>טובה</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td></td>\n",
       "      <td>False</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12615</th>\n",
       "      <td>706</td>\n",
       "      <td>4</td>\n",
       "      <td>אך</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td></td>\n",
       "      <td>False</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12616</th>\n",
       "      <td>706</td>\n",
       "      <td>5</td>\n",
       "      <td>לא</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td></td>\n",
       "      <td>False</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12617</th>\n",
       "      <td>706</td>\n",
       "      <td>6</td>\n",
       "      <td>התרופה</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td></td>\n",
       "      <td>False</td>\n",
       "      <td>2</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12618</th>\n",
       "      <td>706</td>\n",
       "      <td>7</td>\n",
       "      <td>.</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td></td>\n",
       "      <td>False</td>\n",
       "      <td>1</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       sent_id  token_id token_str  token_unk  morph_unk  lemma_unk unk_type  \\\n",
       "12614      706         3      טובה      False      False      False            \n",
       "12615      706         4        אך      False      False      False            \n",
       "12616      706         5        לא      False      False      False            \n",
       "12617      706         6    התרופה      False      False      False            \n",
       "12618      706         7         .      False      False      False            \n",
       "\n",
       "       has_ner  morpheme_count  token_id_rep  \n",
       "12614    False               1             3  \n",
       "12615    False               1             4  \n",
       "12616    False               1             5  \n",
       "12617    False               2             6  \n",
       "12618    False               1             7  "
      ]
     },
     "execution_count": 190,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_unk.tail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 191,
   "metadata": {},
   "outputs": [],
   "source": [
    "dev_gold['ooev_tok'] = ~dev_gold.form.isin(emb_words['tok'])\n",
    "dev_gold['ooev_yap'] = ~dev_gold.form.isin(emb_words['yap'])\n",
    "\n",
    "test_gold['ooev_tok'] = ~test_gold.form.isin(emb_words['tok'])\n",
    "test_gold['ooev_yap'] = ~test_gold.form.isin(emb_words['yap'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 192,
   "metadata": {},
   "outputs": [],
   "source": [
    "dev_gold['ooev_word'] = True\n",
    "test_gold['ooev_word'] = True\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 193,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_df_from_sents(sents):\n",
    "    for_df = []\n",
    "    for i, line in sents.iteritems():\n",
    "        sent_id=i+1\n",
    "        for j, (tok, bio) in enumerate(line):\n",
    "            for_df.append((sent_id, j+1, bio))\n",
    "    df = pd.DataFrame(for_df, columns=['sent_id', 'id', 'biose_layer0'])\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 194,
   "metadata": {},
   "outputs": [],
   "source": [
    "yap_output_folder = 'final_setup/pruned/yap_output'\n",
    "\n",
    "txt_map = defaultdict(dict)\n",
    "\n",
    "for file in os.scandir(yap_output_folder):\n",
    "    if file.name in ('.ipynb_checkpoints', '.conll', '.seg', '.map'):\n",
    "        continue\n",
    "    ds, unit, arch, w_embed, seed_num, output_type = file.name.split('.')\n",
    "    if '_tok' in w_embed:\n",
    "        w_embed = w_embed.replace('_tok', '_yap')\n",
    "    else:\n",
    "        w_embed = w_embed.replace('_yap', '_tok')\n",
    "    txt_map[(ds, arch, w_embed, output_type)] = file.path\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 195,
   "metadata": {},
   "outputs": [],
   "source": [
    "from functools import lru_cache\n",
    "\n",
    "@lru_cache(32)\n",
    "def get_matching_yap_out(ds_sub, ds, arch, w_embed):\n",
    "    if ds_sub=='yap':\n",
    "        yap_out = bclm.read_yap_output(treebank_set=ds)\n",
    "    if ds_sub=='pruned':\n",
    "        yap_out = bclm.read_yap_output(treebank_set=None,\n",
    "                                       tokens_path=bclm.TREEBANK_TOKEN_PATHS[ds], \n",
    "                                        dep_path=txt_map[(ds, arch, w_embed, 'conll')],\n",
    "                                        map_path=txt_map[(ds, arch, w_embed, 'map')],)\n",
    "    return yap_out\n",
    "\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 196,
   "metadata": {},
   "outputs": [],
   "source": [
    "gold_dfs = {'dev': dev_gold, 'test': test_gold}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 197,
   "metadata": {},
   "outputs": [],
   "source": [
    "unks = {'dev': dev_unk, 'test': test_unk}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 198,
   "metadata": {},
   "outputs": [],
   "source": [
    "emb_words['word'] = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 519,
   "metadata": {},
   "outputs": [],
   "source": [
    "scores2 = pickle.load(open('final_setup/oov_scores.pkl', 'rb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 521,
   "metadata": {},
   "outputs": [],
   "source": [
    "new_scores2 = {}\n",
    "for sc_id in scores2:\n",
    "    if not 'test' in sc_id[0]:\n",
    "        new_scores2[sc_id] = scores2[sc_id]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 524,
   "metadata": {},
   "outputs": [],
   "source": [
    "scores2 = new_scores2"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "source": [
    "output_folder = 'final_setup/decode_output'\n",
    "cols = ['form', 'biose_layer0', 'unk_type', 'morpheme_count', 'ooev_tok_part']\n",
    "\n",
    "for file in os.scandir(output_folder):\n",
    "    if file.name=='.ipynb_checkpoints':\n",
    "        continue\n",
    "        \n",
    "    sc_id = tuple(file.name.split('.')[:-1])\n",
    "    gold_name, inp, arch, w_embed, seed_num = sc_id\n",
    "    emb_unit = w_embed.split('_')[-1]\n",
    "\n",
    "    if sc_id not in scores:\n",
    "        print(file.path)\n",
    "        scores[sc_id] = {}\n",
    "        ner_df = get_df_from_sents(nem.read_file_sents(file.path))\n",
    "\n",
    "        if len(gold_name.split('_'))>2:\n",
    "            unit, pred_set, ps_sub = gold_name.split('_')\n",
    "            \n",
    "            if ps_sub in ('pruned', 'yap'):\n",
    "                yap_out = get_matching_yap_out(ps_sub, pred_set, arch, w_embed)\n",
    "                yap_out['ooev'] = ~yap_out.form.isin(emb_words[emb_unit])\n",
    "                full_df = yap_out.merge(ner_df)\n",
    "\n",
    "            elif ps_sub == 'gold':\n",
    "                full_df = gold_dfs[pred_set].copy()\n",
    "                full_df['biose_layer0'] = ner_df.biose_layer0\n",
    "                full_df['ooev'] = full_df['ooev_'+emb_unit]\n",
    "                \n",
    "            full_df = full_df.merge(unks[pred_set], how='left')\n",
    "            ooev_tok_part = (full_df.groupby(['sent_id', 'token_id'])\n",
    "                             .ooev.apply(lambda x: x.any())\n",
    "                             .reset_index().rename(columns={'ooev': 'ooev_tok_part'}))\n",
    "\n",
    "            full_df = full_df.merge(ooev_tok_part, how='left')\n",
    "            gold_ooev = gold_dfs[pred_set].merge(ooev_tok_part, how='left')\n",
    "            token_unk_sents = full_df.groupby('sent_id').apply(get_sent, cols)\n",
    "            token_unk_sents_fixed = get_sents_fixed(token_unk_sents)\n",
    "            ments = [sent_to_mentions_dict(sent, sent_id, ment_len=True) \n",
    "                     for sent_id, sent \n",
    "                     in token_unk_sents_fixed]\n",
    "            ment_set = get_ment_set(ments)\n",
    "            gold_sents = gold_ooev.groupby('sent_id').apply(get_sent, cols)\n",
    "            gold_sents_fixed = get_sents_fixed(gold_sents)\n",
    "            gold_ments = [sent_to_mentions_dict(sent, sent_id, ment_len=True) \n",
    "                          for sent_id, sent \n",
    "                          in gold_sents_fixed]\n",
    "            gold_ment_set = get_ment_set(gold_ments)\n",
    "            for k in ment_set.keys() | gold_ment_set.keys():\n",
    "                scores[sc_id][k] = nem.evaluate_mentions(gold_ment_set[k], \n",
    "                                                        ment_set[k], \n",
    "                                                        verbose=False, \n",
    "                                                        return_tpc=True)\n",
    "\n",
    "        else:\n",
    "            unit, pred_set = gold_name.split('_')\n",
    "            #print ('token')\n",
    "        #p, r, f = nem.evaluate_files(gold_path, file)\n",
    "        #scores[(gold_name, inp, arch, w_embed, seed_num)] = (p, r, f)\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## TOKEN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 199,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/nlp/danb/anaconda3/envs/tf/lib/python3.7/site-packages/pandas/core/frame.py:3509: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  self[k1] = value[k2]\n"
     ]
    }
   ],
   "source": [
    "tok_dev_gold = (bclm\n",
    "                .get_token_df(spdf[spdf.set=='dev'], biose=['biose_layer0'], add_set=False))\n",
    "tok_dev_gold = tok_dev_gold.merge(dev_unk, how='left')\n",
    "tok_test_gold = bclm.get_token_df(spdf[spdf.set=='test'], biose=['biose_layer0'], add_set=False)\n",
    "tok_test_gold['sent_id'] = tok_test_gold.sent_id.map(test_sent_id_map)\n",
    "tok_test_gold = tok_test_gold.merge(test_unk, how='left')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 200,
   "metadata": {},
   "outputs": [],
   "source": [
    "tok_dev_gold['ooev_tok'] = ~tok_dev_gold.token_str.isin(emb_words['tok'])\n",
    "tok_dev_gold['ooev_yap'] = ~tok_dev_gold.token_str.isin(emb_words['yap'])\n",
    "\n",
    "tok_test_gold['ooev_tok'] = ~tok_test_gold.token_str.isin(emb_words['tok'])\n",
    "tok_test_gold['ooev_yap'] = ~tok_test_gold.token_str.isin(emb_words['yap'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 201,
   "metadata": {},
   "outputs": [],
   "source": [
    "tok_dev_gold['ooev_word'] = True\n",
    "tok_test_gold['ooev_word'] = True\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 202,
   "metadata": {},
   "outputs": [],
   "source": [
    "tok_gold_dfs = {'dev': tok_dev_gold, 'test': tok_test_gold}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 204,
   "metadata": {},
   "outputs": [],
   "source": [
    "tok_cols = ['token_str', 'biose_layer0', 'unk_type', 'morpheme_count', 'ooev', 'token_id_rep']\n",
    "\n",
    "tok_gold_ment_sets = {}\n",
    "for pred_set, df in tok_gold_dfs.items():\n",
    "    for emb_unit in ('tok', 'yap', 'word'):\n",
    "        tok_gold_dfs[pred_set]['ooev'] = tok_gold_dfs[pred_set]['ooev_'+emb_unit]\n",
    "        gold_sents = tok_gold_dfs[pred_set].groupby('sent_id').apply(get_sent, tok_cols)\n",
    "        gold_sents_fixed = get_sents_fixed(gold_sents)\n",
    "        tok_gold_ments = [sent_to_mentions_dict(sent, sent_id, ment_len=True, ment_cat=True) \n",
    "                      for sent_id, sent \n",
    "                      in gold_sents_fixed]\n",
    "        gold_ment_set = get_ment_set(tok_gold_ments)\n",
    "        tok_gold_ment_sets[(pred_set, emb_unit)] = gold_ment_set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 222,
   "metadata": {},
   "outputs": [],
   "source": [
    "scores5 = {}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "final_setup/decode_output/morph_dev_gold.morph.char_cnn.ft_yap.44_seed.bmes\n",
      "final_setup/decode_output/morph_dev_yap.morph.char_cnn.ft_yap.44_seed.bmes\n",
      "final_setup/decode_output/morph_test_gold.morph.char_cnn.ft_yap.44_seed.bmes\n",
      "final_setup/decode_output/morph_dev_gold.morph.char_cnn.ft_oov_tok.44_seed.bmes\n",
      "final_setup/decode_output/morph_test_yap.morph.char_cnn.ft_yap.44_seed.bmes\n",
      "final_setup/decode_output/morph_dev_yap.morph.char_cnn.ft_oov_tok.44_seed.bmes\n",
      "final_setup/decode_output/morph_test_gold.morph.char_cnn.ft_oov_tok.44_seed.bmes\n",
      "final_setup/decode_output/morph_dev_gold.morph.char_cnn.glv_tok.45_seed.bmes\n",
      "final_setup/decode_output/morph_dev_yap.morph.char_cnn.glv_tok.45_seed.bmes\n",
      "final_setup/decode_output/morph_test_yap.morph.char_cnn.ft_oov_tok.44_seed.bmes\n",
      "final_setup/decode_output/morph_test_gold.morph.char_cnn.glv_tok.45_seed.bmes\n",
      "final_setup/decode_output/token_dev.multitok.char_cnn.glv_yap.44_seed.bmes\n",
      "final_setup/decode_output/token_test.multitok.char_cnn.glv_yap.44_seed.bmes\n",
      "final_setup/decode_output/morph_test_yap.morph.char_cnn.glv_tok.45_seed.bmes\n",
      "final_setup/decode_output/token_dev.multitok.char_cnn.glv_tok.44_seed.bmes\n",
      "final_setup/decode_output/morph_dev_gold.morph.char_cnn.ft_oov_yap.44_seed.bmes\n",
      "final_setup/decode_output/token_dev.multitok.char_cnn.no_word.44_seed.bmes\n",
      "final_setup/decode_output/token_test.multitok.char_cnn.glv_tok.44_seed.bmes\n",
      "final_setup/decode_output/token_test.multitok.char_cnn.no_word.44_seed.bmes\n",
      "final_setup/decode_output/morph_dev_yap.morph.char_cnn.ft_oov_yap.44_seed.bmes\n",
      "final_setup/decode_output/token_dev.multitok.char_cnn.ft_oov_yap.44_seed.bmes\n",
      "final_setup/decode_output/morph_test_gold.morph.char_cnn.ft_oov_yap.44_seed.bmes\n",
      "final_setup/decode_output/token_dev.multitok.char_cnn.ft_tok.44_seed.bmes\n",
      "final_setup/decode_output/token_test.multitok.char_cnn.ft_oov_yap.44_seed.bmes\n",
      "final_setup/decode_output/morph_test_yap.morph.char_cnn.ft_oov_yap.44_seed.bmes\n",
      "final_setup/decode_output/token_test.multitok.char_cnn.ft_tok.44_seed.bmes\n",
      "final_setup/decode_output/token_dev.multitok.char_cnn.ft_oov_tok.44_seed.bmes\n",
      "final_setup/decode_output/morph_dev_gold.morph.char_cnn.ft_tok.44_seed.bmes\n",
      "final_setup/decode_output/token_dev.multitok.char_lstm.ft_oov_tok.44_seed.bmes\n",
      "final_setup/decode_output/token_test.multitok.char_cnn.ft_oov_tok.44_seed.bmes\n",
      "final_setup/decode_output/morph_dev_yap.morph.char_cnn.ft_tok.44_seed.bmes\n",
      "final_setup/decode_output/token_test.multitok.char_lstm.ft_oov_tok.44_seed.bmes\n",
      "final_setup/decode_output/token_dev.multitok.char_lstm.ft_yap.44_seed.bmes\n",
      "final_setup/decode_output/morph_test_gold.morph.char_cnn.ft_tok.44_seed.bmes\n",
      "final_setup/decode_output/morph_dev_gold.morph.char_cnn.glv_yap.44_seed.bmes\n",
      "final_setup/decode_output/token_dev.multitok.char_lstm.ft_oov_yap.44_seed.bmes\n",
      "final_setup/decode_output/token_test.multitok.char_lstm.ft_yap.44_seed.bmes\n",
      "final_setup/decode_output/morph_test_yap.morph.char_cnn.ft_tok.44_seed.bmes\n",
      "final_setup/decode_output/token_test.multitok.char_lstm.ft_oov_yap.44_seed.bmes\n",
      "final_setup/decode_output/morph_dev_yap.morph.char_cnn.glv_yap.44_seed.bmes\n",
      "final_setup/decode_output/token_dev.multitok.char_lstm.ft_tok.44_seed.bmes\n",
      "final_setup/decode_output/morph_test_gold.morph.char_cnn.glv_yap.44_seed.bmes\n",
      "final_setup/decode_output/morph_dev_gold.morph.no_char.ft_tok.45_seed.bmes\n",
      "final_setup/decode_output/token_test.multitok.char_lstm.ft_tok.44_seed.bmes\n",
      "final_setup/decode_output/morph_dev_yap.morph.no_char.ft_tok.45_seed.bmes\n",
      "final_setup/decode_output/morph_test_yap.morph.char_cnn.glv_yap.44_seed.bmes\n",
      "final_setup/decode_output/token_dev.multitok.char_lstm.glv_tok.44_seed.bmes\n",
      "final_setup/decode_output/morph_test_gold.morph.no_char.ft_tok.45_seed.bmes\n",
      "final_setup/decode_output/morph_dev_gold.morph.no_char.ft_oov_tok.44_seed.bmes\n",
      "final_setup/decode_output/token_test.multitok.char_lstm.glv_tok.44_seed.bmes\n",
      "final_setup/decode_output/token_dev.multitok.char_cnn.glv_yap.45_seed.bmes\n",
      "final_setup/decode_output/morph_test_yap.morph.no_char.ft_tok.45_seed.bmes\n",
      "final_setup/decode_output/morph_dev_yap.morph.no_char.ft_oov_tok.44_seed.bmes\n",
      "final_setup/decode_output/token_dev.multitok.char_cnn.no_word.45_seed.bmes\n",
      "final_setup/decode_output/token_test.multitok.char_cnn.glv_yap.45_seed.bmes\n",
      "final_setup/decode_output/morph_test_gold.morph.no_char.ft_oov_tok.44_seed.bmes\n",
      "final_setup/decode_output/token_dev.multitok.char_cnn.ft_tok.45_seed.bmes\n",
      "final_setup/decode_output/token_test.multitok.char_cnn.no_word.45_seed.bmes\n",
      "final_setup/decode_output/token_test.multitok.char_cnn.ft_tok.45_seed.bmes\n",
      "final_setup/decode_output/token_dev.multitok.char_cnn.ft_oov_tok.45_seed.bmes\n",
      "final_setup/decode_output/morph_test_yap.morph.no_char.ft_oov_tok.44_seed.bmes\n",
      "final_setup/decode_output/morph_dev_gold.morph.char_lstm.ft_oov_yap.44_seed.bmes\n",
      "final_setup/decode_output/token_dev.token.char_cnn.ft_tok.44_seed.bmes\n",
      "final_setup/decode_output/morph_dev_yap.morph.char_lstm.ft_oov_yap.44_seed.bmes\n",
      "final_setup/decode_output/token_test.multitok.char_cnn.ft_oov_tok.45_seed.bmes\n",
      "final_setup/decode_output/token_dev.multitok.char_cnn.glv_tok.45_seed.bmes\n",
      "final_setup/decode_output/morph_dev_gold.morph.char_lstm.ft_tok.44_seed.bmes\n",
      "final_setup/decode_output/token_test.multitok.char_cnn.glv_tok.45_seed.bmes\n",
      "final_setup/decode_output/token_dev.multitok.char_cnn.ft_oov_yap.45_seed.bmes\n",
      "final_setup/decode_output/morph_test_gold.morph.char_lstm.ft_oov_yap.44_seed.bmes\n",
      "final_setup/decode_output/morph_test_yap.morph.char_lstm.ft_oov_yap.44_seed.bmes\n",
      "final_setup/decode_output/morph_dev_yap.morph.char_lstm.ft_tok.44_seed.bmes\n",
      "final_setup/decode_output/token_test.multitok.char_cnn.ft_oov_yap.45_seed.bmes\n",
      "final_setup/decode_output/token_dev.multitok.char_lstm.ft_yap.45_seed.bmes\n",
      "final_setup/decode_output/token_dev.multitok.char_lstm.ft_oov_yap.45_seed.bmes\n",
      "final_setup/decode_output/morph_test_gold.morph.char_lstm.ft_tok.44_seed.bmes\n",
      "final_setup/decode_output/morph_dev_gold.morph.no_char.ft_oov_yap.44_seed.bmes\n",
      "final_setup/decode_output/token_test.multitok.char_lstm.ft_yap.45_seed.bmes\n",
      "final_setup/decode_output/token_dev.multitok.char_lstm.ft_oov_tok.45_seed.bmes\n",
      "final_setup/decode_output/token_test.multitok.char_lstm.ft_oov_yap.45_seed.bmes\n",
      "final_setup/decode_output/morph_dev_yap.morph.no_char.ft_oov_yap.44_seed.bmes\n",
      "final_setup/decode_output/morph_test_yap.morph.char_lstm.ft_tok.44_seed.bmes\n",
      "final_setup/decode_output/token_dev.multitok.char_lstm.ft_tok.45_seed.bmes\n",
      "final_setup/decode_output/token_test.multitok.char_lstm.ft_oov_tok.45_seed.bmes\n",
      "final_setup/decode_output/morph_dev_gold.morph.char_lstm.glv_tok.44_seed.bmes\n",
      "final_setup/decode_output/morph_test_gold.morph.no_char.ft_oov_yap.44_seed.bmes\n",
      "final_setup/decode_output/token_test.multitok.char_lstm.ft_tok.45_seed.bmes\n",
      "final_setup/decode_output/token_dev.multitok.char_cnn.no_word.46_seed.bmes\n",
      "final_setup/decode_output/morph_dev_yap.morph.char_lstm.glv_tok.44_seed.bmes\n",
      "final_setup/decode_output/morph_test_yap.morph.no_char.ft_oov_yap.44_seed.bmes\n",
      "final_setup/decode_output/token_dev.multitok.char_lstm.glv_tok.45_seed.bmes\n",
      "final_setup/decode_output/token_test.multitok.char_cnn.no_word.46_seed.bmes\n",
      "final_setup/decode_output/morph_dev_gold.morph.no_char.glv_yap.44_seed.bmes\n",
      "final_setup/decode_output/morph_test_gold.morph.char_lstm.glv_tok.44_seed.bmes\n",
      "final_setup/decode_output/token_dev.multitok.char_cnn.ft_oov_tok.46_seed.bmes\n",
      "final_setup/decode_output/token_test.multitok.char_lstm.glv_tok.45_seed.bmes\n",
      "final_setup/decode_output/morph_dev_yap.morph.no_char.glv_yap.44_seed.bmes\n",
      "final_setup/decode_output/morph_test_yap.morph.char_lstm.glv_tok.44_seed.bmes\n",
      "final_setup/decode_output/token_dev.multitok.char_cnn.glv_yap.46_seed.bmes\n",
      "final_setup/decode_output/token_test.multitok.char_cnn.ft_oov_tok.46_seed.bmes\n",
      "final_setup/decode_output/morph_test_gold.morph.no_char.glv_yap.44_seed.bmes\n",
      "final_setup/decode_output/token_dev.multitok.char_cnn.ft_oov_yap.46_seed.bmes\n",
      "final_setup/decode_output/token_test.multitok.char_cnn.glv_yap.46_seed.bmes\n",
      "final_setup/decode_output/morph_dev_gold.morph.char_lstm.no_word.44_seed.bmes\n",
      "final_setup/decode_output/morph_test_yap.morph.no_char.glv_yap.44_seed.bmes\n",
      "final_setup/decode_output/token_dev.multitok.char_cnn.glv_tok.46_seed.bmes\n",
      "final_setup/decode_output/token_test.multitok.char_cnn.ft_oov_yap.46_seed.bmes\n",
      "final_setup/decode_output/morph_dev_yap.morph.char_lstm.no_word.44_seed.bmes\n",
      "final_setup/decode_output/token_dev.multitok.char_cnn.ft_tok.46_seed.bmes\n",
      "final_setup/decode_output/token_test.multitok.char_cnn.glv_tok.46_seed.bmes\n",
      "final_setup/decode_output/token_dev.multitok.char_lstm.ft_yap.46_seed.bmes\n",
      "final_setup/decode_output/morph_test_gold.morph.char_lstm.no_word.44_seed.bmes\n",
      "final_setup/decode_output/token_test.multitok.char_cnn.ft_tok.46_seed.bmes\n",
      "final_setup/decode_output/token_dev.multitok.char_lstm.ft_tok.46_seed.bmes\n",
      "final_setup/decode_output/morph_test_yap.morph.char_lstm.no_word.44_seed.bmes\n",
      "final_setup/decode_output/token_test.multitok.char_lstm.ft_yap.46_seed.bmes\n",
      "final_setup/decode_output/token_dev.multitok.char_lstm.ft_oov_yap.46_seed.bmes\n",
      "final_setup/decode_output/token_test.multitok.char_lstm.ft_tok.46_seed.bmes\n",
      "final_setup/decode_output/token_dev.multitok.char_lstm.ft_oov_tok.46_seed.bmes\n",
      "final_setup/decode_output/token_test.multitok.char_lstm.ft_oov_yap.46_seed.bmes\n",
      "final_setup/decode_output/morph_dev_gold.morph.char_cnn.ft_yap.45_seed.bmes\n",
      "final_setup/decode_output/token_test.multitok.char_lstm.ft_oov_tok.46_seed.bmes\n",
      "final_setup/decode_output/token_dev.token.char_cnn.ft_oov_yap.44_seed.bmes\n",
      "final_setup/decode_output/morph_dev_yap.morph.char_cnn.ft_yap.45_seed.bmes\n",
      "final_setup/decode_output/token_dev.multitok.no_char.ft_oov_tok.44_seed.bmes\n",
      "final_setup/decode_output/morph_dev_gold.morph.char_cnn.ft_oov_tok.45_seed.bmes\n",
      "final_setup/decode_output/token_test.token.char_cnn.ft_oov_yap.44_seed.bmes\n",
      "final_setup/decode_output/token_dev.multitok.no_char.ft_oov_yap.44_seed.bmes\n",
      "final_setup/decode_output/morph_test_gold.morph.char_cnn.ft_yap.45_seed.bmes\n",
      "final_setup/decode_output/morph_dev_yap.morph.char_cnn.ft_oov_tok.45_seed.bmes\n",
      "final_setup/decode_output/token_test.multitok.no_char.ft_oov_tok.44_seed.bmes\n"
     ]
    }
   ],
   "source": [
    "output_folder = 'final_setup/decode_output'\n",
    "cols = ['form', 'biose_layer0', 'unk_type', 'morpheme_count', 'ooev_tok_part', 'token_id_rep']\n",
    "tok_cols = ['token_str', 'biose_layer0', 'unk_type', 'morpheme_count', 'ooev', 'token_id_rep']\n",
    "\n",
    "for file in os.scandir(output_folder):\n",
    "    if file.name=='.ipynb_checkpoints':\n",
    "        continue\n",
    "        \n",
    "    sc_id = tuple(file.name.split('.')[:-1])\n",
    "    gold_name, inp, arch, w_embed, seed_num = sc_id\n",
    "    emb_unit = w_embed.split('_')[-1]\n",
    "\n",
    "    if sc_id not in scores5:\n",
    "        print(file.path)\n",
    "        scores5[sc_id] = {}\n",
    "        ner_df = get_df_from_sents(nem.read_file_sents(file.path))\n",
    "\n",
    "        if len(gold_name.split('_'))>2:\n",
    "            unit, pred_set, ps_sub = gold_name.split('_')\n",
    "            \n",
    "            if ps_sub in ('pruned', 'yap'):\n",
    "                yap_out = get_matching_yap_out(ps_sub, pred_set, arch, w_embed)\n",
    "                yap_out['ooev'] = ~yap_out.form.isin(emb_words[emb_unit])\n",
    "                full_df = yap_out.merge(ner_df)\n",
    "\n",
    "            elif ps_sub == 'gold':\n",
    "                full_df = gold_dfs[pred_set].copy()\n",
    "                full_df['biose_layer0'] = ner_df.biose_layer0\n",
    "                full_df['ooev'] = full_df['ooev_'+emb_unit]\n",
    "                \n",
    "            full_df = full_df.merge(unks[pred_set], how='left')\n",
    "            ooev_tok_part = (full_df.groupby(['sent_id', 'token_id'])\n",
    "                             .ooev.apply(lambda x: x.any())\n",
    "                             .reset_index().rename(columns={'ooev': 'ooev_tok_part'}))\n",
    "\n",
    "            full_df = full_df.merge(ooev_tok_part, how='left')\n",
    "            gold_ooev = gold_dfs[pred_set].merge(ooev_tok_part, how='left')\n",
    "            token_unk_sents = full_df.groupby('sent_id').apply(get_sent, cols)\n",
    "            token_unk_sents_fixed = get_sents_fixed(token_unk_sents)\n",
    "            ments = [sent_to_mentions_dict(sent, sent_id, \n",
    "                                           ment_len=True, ment_cat=True) \n",
    "                     for sent_id, sent \n",
    "                     in token_unk_sents_fixed]\n",
    "            ment_set = get_ment_set(ments)\n",
    "            gold_sents = gold_ooev.groupby('sent_id').apply(get_sent, cols)\n",
    "            gold_sents_fixed = get_sents_fixed(gold_sents)\n",
    "            gold_ments = [sent_to_mentions_dict(sent, sent_id, \n",
    "                                                ment_len=True, ment_cat=True) \n",
    "                          for sent_id, sent \n",
    "                          in gold_sents_fixed]\n",
    "            gold_ment_set = get_ment_set(gold_ments)\n",
    "            for k in ment_set.keys() | gold_ment_set.keys():\n",
    "                scores5[sc_id][k] = nem.evaluate_mentions(gold_ment_set[k], \n",
    "                                                        ment_set[k], \n",
    "                                                        verbose=False, \n",
    "                                                        return_tpc=True)\n",
    "\n",
    "        ##TOKEN\n",
    "        else:\n",
    "            ner_df = ner_df.rename(columns= {'id': 'token_id'})\n",
    "            unit, pred_set = gold_name.split('_')\n",
    "            full_df = tok_gold_dfs[pred_set].copy()\n",
    "            full_df['biose_layer0'] = ner_df.biose_layer0\n",
    "            full_df['ooev'] = full_df['ooev_'+emb_unit]\n",
    "                \n",
    "            token_unk_sents = full_df.groupby('sent_id').apply(get_sent, tok_cols)\n",
    "            token_unk_sents_fixed = get_sents_fixed(token_unk_sents)\n",
    "            ments = [sent_to_mentions_dict(sent, sent_id, \n",
    "                                           ment_len=True, ment_cat=True) \n",
    "                     for sent_id, sent \n",
    "                     in token_unk_sents_fixed]\n",
    "            ment_set = get_ment_set(ments)\n",
    "\n",
    "            gold_ment_set = tok_gold_ment_sets[(pred_set, emb_unit)]\n",
    "            for k in ment_set.keys() | gold_ment_set.keys():\n",
    "                scores5[sc_id][k] = nem.evaluate_mentions(gold_ment_set[k], \n",
    "                                                        ment_set[k], \n",
    "                                                        verbose=False, \n",
    "                                                        return_tpc=True)\n",
    "            \n",
    "\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 225,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "pickle.dump(scores5, open('final_setup/oov_scores5.pkl', 'wb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 224,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2100"
      ]
     },
     "execution_count": 224,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(scores5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 226,
   "metadata": {},
   "outputs": [],
   "source": [
    "def prf_from_tpc(t, p, c):\n",
    "    if p==0:\n",
    "        prec=-1\n",
    "    else:\n",
    "        prec = c / p\n",
    "    \n",
    "    if t==0:\n",
    "        recall=-1\n",
    "    else:\n",
    "        recall = c / t\n",
    "\n",
    "    if prec+recall==0:\n",
    "        f1=-1\n",
    "    else:\n",
    "        f1 = 2*prec*recall/(prec+recall)    \n",
    "    return prec, recall, f1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 238,
   "metadata": {},
   "outputs": [],
   "source": [
    "same = lambda x: x\n",
    "\n",
    "def ignore(x):\n",
    "    return 'all'\n",
    "\n",
    "four_up = lambda x: x if x<=3 else 4\n",
    "\n",
    "def ut_gm(x):\n",
    "    filt_x = list(filter(lambda a: a != '', x))\n",
    "    return tuple(sorted(set(filt_x)))\n",
    "\n",
    "lenset = lambda x: min(len(set(x)), 4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 259,
   "metadata": {},
   "outputs": [],
   "source": [
    "scid_names = ['gold_name', 'unit', 'arch', 'w_embed', 'seed_num']\n",
    "def group_scores(scs, groupers, names = ['unk_type', 'morpheme_count', 'ooev', 'token_id_rep'], \n",
    "                 ment_grouper=same, ment_cat_grouper=ignore):\n",
    "    grouped_scs = []\n",
    "    for sc_id, keys in scs.items():\n",
    "        single_gsc = defaultdict(lambda: [0, 0, 0])\n",
    "        for k in keys:\n",
    "            score = scs[sc_id][k]\n",
    "            (umo, ment_len), ment_cat = k\n",
    "            ment_len = ment_grouper(ment_len)\n",
    "            ment_cat = ment_cat_grouper(ment_cat)\n",
    "            umo = list(zip(*umo))\n",
    "            grouped_umo = [grouper(x) \n",
    "                                 for grouper, x in zip(groupers, umo)]\n",
    "            grouped_umo.append(ment_len)\n",
    "            grouped_umo.append(ment_cat)\n",
    "            grouped_umo = tuple(grouped_umo)\n",
    "            single_gsc[grouped_umo][0] += score[3]\n",
    "            single_gsc[grouped_umo][1] += score[4]\n",
    "            single_gsc[grouped_umo][2] += score[5]\n",
    "\n",
    "        \n",
    "        gsc = []\n",
    "        for grp_k in single_gsc:\n",
    "            t,p,c = single_gsc[grp_k]\n",
    "            prec, recall, f1 = prf_from_tpc(t,p,c)\n",
    "            if f1==-1:\n",
    "                f1=0\n",
    "            gsc.append(tuple(list(sc_id)+list(grp_k)+[t,p,c, prec, recall, f1]))\n",
    "        grouped_scs.extend(gsc)\n",
    "        \n",
    "    names = scid_names+names+['ment_len', 'ment_cat']+['true','pred','correct', 'p', 'r', 'f']\n",
    "    return pd.DataFrame(grouped_scs, columns=names)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 248,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>gold_name</th>\n",
       "      <th>unit</th>\n",
       "      <th>arch</th>\n",
       "      <th>w_embed</th>\n",
       "      <th>seed_num</th>\n",
       "      <th>unk_type</th>\n",
       "      <th>morpheme_count</th>\n",
       "      <th>ooev</th>\n",
       "      <th>token_id_rep</th>\n",
       "      <th>ment_len</th>\n",
       "      <th>ment_cat</th>\n",
       "      <th>true</th>\n",
       "      <th>pred</th>\n",
       "      <th>correct</th>\n",
       "      <th>p</th>\n",
       "      <th>r</th>\n",
       "      <th>f</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>morph_dev_gold</td>\n",
       "      <td>morph</td>\n",
       "      <td>char_cnn</td>\n",
       "      <td>ft_yap</td>\n",
       "      <td>44_seed</td>\n",
       "      <td>all</td>\n",
       "      <td>all</td>\n",
       "      <td>all</td>\n",
       "      <td>1</td>\n",
       "      <td>all</td>\n",
       "      <td>all</td>\n",
       "      <td>255</td>\n",
       "      <td>224</td>\n",
       "      <td>194</td>\n",
       "      <td>0.866071</td>\n",
       "      <td>0.760784</td>\n",
       "      <td>0.810021</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>morph_dev_gold</td>\n",
       "      <td>morph</td>\n",
       "      <td>char_cnn</td>\n",
       "      <td>ft_yap</td>\n",
       "      <td>44_seed</td>\n",
       "      <td>all</td>\n",
       "      <td>all</td>\n",
       "      <td>all</td>\n",
       "      <td>2</td>\n",
       "      <td>all</td>\n",
       "      <td>all</td>\n",
       "      <td>183</td>\n",
       "      <td>159</td>\n",
       "      <td>141</td>\n",
       "      <td>0.886792</td>\n",
       "      <td>0.770492</td>\n",
       "      <td>0.824561</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>morph_dev_gold</td>\n",
       "      <td>morph</td>\n",
       "      <td>char_cnn</td>\n",
       "      <td>ft_yap</td>\n",
       "      <td>44_seed</td>\n",
       "      <td>all</td>\n",
       "      <td>all</td>\n",
       "      <td>all</td>\n",
       "      <td>4</td>\n",
       "      <td>all</td>\n",
       "      <td>all</td>\n",
       "      <td>15</td>\n",
       "      <td>8</td>\n",
       "      <td>6</td>\n",
       "      <td>0.750000</td>\n",
       "      <td>0.400000</td>\n",
       "      <td>0.521739</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>morph_dev_gold</td>\n",
       "      <td>morph</td>\n",
       "      <td>char_cnn</td>\n",
       "      <td>ft_yap</td>\n",
       "      <td>44_seed</td>\n",
       "      <td>all</td>\n",
       "      <td>all</td>\n",
       "      <td>all</td>\n",
       "      <td>3</td>\n",
       "      <td>all</td>\n",
       "      <td>all</td>\n",
       "      <td>46</td>\n",
       "      <td>35</td>\n",
       "      <td>25</td>\n",
       "      <td>0.714286</td>\n",
       "      <td>0.543478</td>\n",
       "      <td>0.617284</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>morph_dev_yap</td>\n",
       "      <td>morph</td>\n",
       "      <td>char_cnn</td>\n",
       "      <td>ft_yap</td>\n",
       "      <td>44_seed</td>\n",
       "      <td>all</td>\n",
       "      <td>all</td>\n",
       "      <td>all</td>\n",
       "      <td>1</td>\n",
       "      <td>all</td>\n",
       "      <td>all</td>\n",
       "      <td>255</td>\n",
       "      <td>222</td>\n",
       "      <td>166</td>\n",
       "      <td>0.747748</td>\n",
       "      <td>0.650980</td>\n",
       "      <td>0.696017</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        gold_name   unit      arch w_embed seed_num unk_type morpheme_count  \\\n",
       "0  morph_dev_gold  morph  char_cnn  ft_yap  44_seed      all            all   \n",
       "1  morph_dev_gold  morph  char_cnn  ft_yap  44_seed      all            all   \n",
       "2  morph_dev_gold  morph  char_cnn  ft_yap  44_seed      all            all   \n",
       "3  morph_dev_gold  morph  char_cnn  ft_yap  44_seed      all            all   \n",
       "4   morph_dev_yap  morph  char_cnn  ft_yap  44_seed      all            all   \n",
       "\n",
       "  ooev  token_id_rep ment_len ment_cat  true  pred  correct         p  \\\n",
       "0  all             1      all      all   255   224      194  0.866071   \n",
       "1  all             2      all      all   183   159      141  0.886792   \n",
       "2  all             4      all      all    15     8        6  0.750000   \n",
       "3  all             3      all      all    46    35       25  0.714286   \n",
       "4  all             1      all      all   255   222      166  0.747748   \n",
       "\n",
       "          r         f  \n",
       "0  0.760784  0.810021  \n",
       "1  0.770492  0.824561  \n",
       "2  0.400000  0.521739  \n",
       "3  0.543478  0.617284  \n",
       "4  0.650980  0.696017  "
      ]
     },
     "execution_count": 248,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gsc = group_scores(scores5, groupers=[ignore, ignore, ignore, lenset], ment_grouper=ignore)\n",
    "gsc.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 249,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "w_embed\n",
       "ft_oov_tok    1200\n",
       "ft_oov_yap    1200\n",
       "ft_tok        1200\n",
       "ft_yap        1200\n",
       "glv_tok       1200\n",
       "glv_yap       1200\n",
       "no_word       1200\n",
       "dtype: int64"
      ]
     },
     "execution_count": 249,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gsc.groupby('w_embed').size()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 250,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>token_id_rep</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>gold_name</th>\n",
       "      <th>unit</th>\n",
       "      <th>arch</th>\n",
       "      <th>w_embed</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"21\" valign=\"top\">morph_dev_gold</th>\n",
       "      <th rowspan=\"21\" valign=\"top\">morph</th>\n",
       "      <th rowspan=\"7\" valign=\"top\">char_cnn</th>\n",
       "      <th>ft_oov_tok</th>\n",
       "      <td>81.62</td>\n",
       "      <td>83.52</td>\n",
       "      <td>61.51</td>\n",
       "      <td>44.72</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_oov_yap</th>\n",
       "      <td>81.71</td>\n",
       "      <td>83.81</td>\n",
       "      <td>61.97</td>\n",
       "      <td>51.07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_tok</th>\n",
       "      <td>80.98</td>\n",
       "      <td>83.42</td>\n",
       "      <td>61.36</td>\n",
       "      <td>48.60</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_yap</th>\n",
       "      <td>80.02</td>\n",
       "      <td>83.43</td>\n",
       "      <td>61.79</td>\n",
       "      <td>48.80</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_tok</th>\n",
       "      <td>80.26</td>\n",
       "      <td>81.46</td>\n",
       "      <td>59.17</td>\n",
       "      <td>41.30</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_yap</th>\n",
       "      <td>80.14</td>\n",
       "      <td>81.09</td>\n",
       "      <td>60.94</td>\n",
       "      <td>42.80</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>no_word</th>\n",
       "      <td>61.72</td>\n",
       "      <td>69.73</td>\n",
       "      <td>49.43</td>\n",
       "      <td>38.58</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"7\" valign=\"top\">char_lstm</th>\n",
       "      <th>ft_oov_tok</th>\n",
       "      <td>81.15</td>\n",
       "      <td>83.06</td>\n",
       "      <td>61.48</td>\n",
       "      <td>45.70</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_oov_yap</th>\n",
       "      <td>82.21</td>\n",
       "      <td>83.06</td>\n",
       "      <td>59.19</td>\n",
       "      <td>44.97</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_tok</th>\n",
       "      <td>80.37</td>\n",
       "      <td>82.91</td>\n",
       "      <td>60.72</td>\n",
       "      <td>47.90</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_yap</th>\n",
       "      <td>80.47</td>\n",
       "      <td>82.96</td>\n",
       "      <td>58.64</td>\n",
       "      <td>48.54</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_tok</th>\n",
       "      <td>80.09</td>\n",
       "      <td>81.40</td>\n",
       "      <td>60.01</td>\n",
       "      <td>40.98</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_yap</th>\n",
       "      <td>80.42</td>\n",
       "      <td>81.39</td>\n",
       "      <td>64.35</td>\n",
       "      <td>47.10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>no_word</th>\n",
       "      <td>62.47</td>\n",
       "      <td>69.47</td>\n",
       "      <td>44.22</td>\n",
       "      <td>35.83</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"7\" valign=\"top\">no_char</th>\n",
       "      <th>ft_oov_tok</th>\n",
       "      <td>81.44</td>\n",
       "      <td>83.02</td>\n",
       "      <td>62.86</td>\n",
       "      <td>48.97</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_oov_yap</th>\n",
       "      <td>81.67</td>\n",
       "      <td>83.38</td>\n",
       "      <td>60.79</td>\n",
       "      <td>45.21</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_tok</th>\n",
       "      <td>79.19</td>\n",
       "      <td>83.20</td>\n",
       "      <td>63.15</td>\n",
       "      <td>47.92</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_yap</th>\n",
       "      <td>78.61</td>\n",
       "      <td>82.91</td>\n",
       "      <td>61.03</td>\n",
       "      <td>48.50</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_tok</th>\n",
       "      <td>77.82</td>\n",
       "      <td>81.01</td>\n",
       "      <td>59.88</td>\n",
       "      <td>36.58</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_yap</th>\n",
       "      <td>77.06</td>\n",
       "      <td>80.94</td>\n",
       "      <td>61.81</td>\n",
       "      <td>47.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>no_word</th>\n",
       "      <td>60.40</td>\n",
       "      <td>64.79</td>\n",
       "      <td>42.86</td>\n",
       "      <td>28.93</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"21\" valign=\"top\">morph_dev_pruned</th>\n",
       "      <th rowspan=\"21\" valign=\"top\">morph</th>\n",
       "      <th rowspan=\"7\" valign=\"top\">char_cnn</th>\n",
       "      <th>ft_oov_tok</th>\n",
       "      <td>79.67</td>\n",
       "      <td>80.83</td>\n",
       "      <td>59.92</td>\n",
       "      <td>40.28</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_oov_yap</th>\n",
       "      <td>79.93</td>\n",
       "      <td>82.01</td>\n",
       "      <td>59.15</td>\n",
       "      <td>47.67</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_tok</th>\n",
       "      <td>77.16</td>\n",
       "      <td>81.09</td>\n",
       "      <td>58.62</td>\n",
       "      <td>43.03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_yap</th>\n",
       "      <td>77.96</td>\n",
       "      <td>81.56</td>\n",
       "      <td>58.88</td>\n",
       "      <td>45.89</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_tok</th>\n",
       "      <td>77.50</td>\n",
       "      <td>78.61</td>\n",
       "      <td>56.60</td>\n",
       "      <td>38.09</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_yap</th>\n",
       "      <td>78.14</td>\n",
       "      <td>78.97</td>\n",
       "      <td>58.96</td>\n",
       "      <td>41.93</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>no_word</th>\n",
       "      <td>57.28</td>\n",
       "      <td>66.14</td>\n",
       "      <td>46.98</td>\n",
       "      <td>34.34</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"7\" valign=\"top\">char_lstm</th>\n",
       "      <th>ft_oov_tok</th>\n",
       "      <td>78.29</td>\n",
       "      <td>81.06</td>\n",
       "      <td>59.23</td>\n",
       "      <td>39.50</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_oov_yap</th>\n",
       "      <td>80.20</td>\n",
       "      <td>82.04</td>\n",
       "      <td>56.81</td>\n",
       "      <td>43.09</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_tok</th>\n",
       "      <td>76.40</td>\n",
       "      <td>80.26</td>\n",
       "      <td>58.77</td>\n",
       "      <td>43.64</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_yap</th>\n",
       "      <td>78.18</td>\n",
       "      <td>81.13</td>\n",
       "      <td>56.55</td>\n",
       "      <td>45.73</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_tok</th>\n",
       "      <td>76.40</td>\n",
       "      <td>79.38</td>\n",
       "      <td>55.76</td>\n",
       "      <td>40.74</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_yap</th>\n",
       "      <td>78.06</td>\n",
       "      <td>79.21</td>\n",
       "      <td>62.96</td>\n",
       "      <td>41.23</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>no_word</th>\n",
       "      <td>58.56</td>\n",
       "      <td>65.64</td>\n",
       "      <td>40.59</td>\n",
       "      <td>35.13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"7\" valign=\"top\">no_char</th>\n",
       "      <th>ft_oov_tok</th>\n",
       "      <td>78.49</td>\n",
       "      <td>81.65</td>\n",
       "      <td>62.66</td>\n",
       "      <td>40.49</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_oov_yap</th>\n",
       "      <td>79.34</td>\n",
       "      <td>80.48</td>\n",
       "      <td>55.65</td>\n",
       "      <td>42.97</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_tok</th>\n",
       "      <td>75.35</td>\n",
       "      <td>80.62</td>\n",
       "      <td>61.78</td>\n",
       "      <td>45.46</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_yap</th>\n",
       "      <td>77.15</td>\n",
       "      <td>81.34</td>\n",
       "      <td>57.37</td>\n",
       "      <td>47.09</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_tok</th>\n",
       "      <td>75.22</td>\n",
       "      <td>78.62</td>\n",
       "      <td>56.53</td>\n",
       "      <td>37.28</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_yap</th>\n",
       "      <td>74.61</td>\n",
       "      <td>78.50</td>\n",
       "      <td>58.54</td>\n",
       "      <td>43.29</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>no_word</th>\n",
       "      <td>55.22</td>\n",
       "      <td>61.11</td>\n",
       "      <td>39.53</td>\n",
       "      <td>25.43</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"21\" valign=\"top\">morph_dev_yap</th>\n",
       "      <th rowspan=\"21\" valign=\"top\">morph</th>\n",
       "      <th rowspan=\"7\" valign=\"top\">char_cnn</th>\n",
       "      <th>ft_oov_tok</th>\n",
       "      <td>71.40</td>\n",
       "      <td>78.95</td>\n",
       "      <td>57.32</td>\n",
       "      <td>44.16</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_oov_yap</th>\n",
       "      <td>71.88</td>\n",
       "      <td>78.99</td>\n",
       "      <td>56.54</td>\n",
       "      <td>48.04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_tok</th>\n",
       "      <td>70.63</td>\n",
       "      <td>78.23</td>\n",
       "      <td>55.63</td>\n",
       "      <td>47.61</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_yap</th>\n",
       "      <td>69.94</td>\n",
       "      <td>78.03</td>\n",
       "      <td>55.34</td>\n",
       "      <td>46.17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_tok</th>\n",
       "      <td>70.52</td>\n",
       "      <td>75.43</td>\n",
       "      <td>53.24</td>\n",
       "      <td>40.58</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_yap</th>\n",
       "      <td>70.44</td>\n",
       "      <td>74.87</td>\n",
       "      <td>53.39</td>\n",
       "      <td>40.70</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>no_word</th>\n",
       "      <td>53.41</td>\n",
       "      <td>65.76</td>\n",
       "      <td>43.54</td>\n",
       "      <td>39.05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"7\" valign=\"top\">char_lstm</th>\n",
       "      <th>ft_oov_tok</th>\n",
       "      <td>70.98</td>\n",
       "      <td>78.16</td>\n",
       "      <td>56.03</td>\n",
       "      <td>40.63</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_oov_yap</th>\n",
       "      <td>72.23</td>\n",
       "      <td>78.26</td>\n",
       "      <td>54.53</td>\n",
       "      <td>41.53</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_tok</th>\n",
       "      <td>70.07</td>\n",
       "      <td>77.21</td>\n",
       "      <td>55.27</td>\n",
       "      <td>44.71</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_yap</th>\n",
       "      <td>69.90</td>\n",
       "      <td>77.22</td>\n",
       "      <td>52.80</td>\n",
       "      <td>44.29</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_tok</th>\n",
       "      <td>69.98</td>\n",
       "      <td>75.40</td>\n",
       "      <td>54.01</td>\n",
       "      <td>42.34</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_yap</th>\n",
       "      <td>70.22</td>\n",
       "      <td>75.55</td>\n",
       "      <td>57.13</td>\n",
       "      <td>44.74</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>no_word</th>\n",
       "      <td>54.28</td>\n",
       "      <td>66.32</td>\n",
       "      <td>39.28</td>\n",
       "      <td>33.29</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"7\" valign=\"top\">no_char</th>\n",
       "      <th>ft_oov_tok</th>\n",
       "      <td>70.91</td>\n",
       "      <td>78.15</td>\n",
       "      <td>58.40</td>\n",
       "      <td>44.89</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_oov_yap</th>\n",
       "      <td>72.10</td>\n",
       "      <td>78.65</td>\n",
       "      <td>55.70</td>\n",
       "      <td>43.03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_tok</th>\n",
       "      <td>69.37</td>\n",
       "      <td>77.63</td>\n",
       "      <td>58.13</td>\n",
       "      <td>43.82</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_yap</th>\n",
       "      <td>68.86</td>\n",
       "      <td>77.83</td>\n",
       "      <td>54.94</td>\n",
       "      <td>45.73</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_tok</th>\n",
       "      <td>68.23</td>\n",
       "      <td>75.84</td>\n",
       "      <td>54.10</td>\n",
       "      <td>36.41</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_yap</th>\n",
       "      <td>67.30</td>\n",
       "      <td>75.38</td>\n",
       "      <td>55.78</td>\n",
       "      <td>46.84</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>no_word</th>\n",
       "      <td>51.81</td>\n",
       "      <td>61.64</td>\n",
       "      <td>40.32</td>\n",
       "      <td>30.74</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"42\" valign=\"top\">token_dev</th>\n",
       "      <th rowspan=\"21\" valign=\"top\">multitok</th>\n",
       "      <th rowspan=\"7\" valign=\"top\">char_cnn</th>\n",
       "      <th>ft_oov_tok</th>\n",
       "      <td>77.80</td>\n",
       "      <td>81.30</td>\n",
       "      <td>61.18</td>\n",
       "      <td>42.78</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_oov_yap</th>\n",
       "      <td>76.27</td>\n",
       "      <td>77.80</td>\n",
       "      <td>60.99</td>\n",
       "      <td>47.07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_tok</th>\n",
       "      <td>74.46</td>\n",
       "      <td>80.64</td>\n",
       "      <td>60.44</td>\n",
       "      <td>42.78</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_yap</th>\n",
       "      <td>71.22</td>\n",
       "      <td>75.84</td>\n",
       "      <td>55.58</td>\n",
       "      <td>42.15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_tok</th>\n",
       "      <td>72.69</td>\n",
       "      <td>77.42</td>\n",
       "      <td>59.08</td>\n",
       "      <td>39.86</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_yap</th>\n",
       "      <td>70.50</td>\n",
       "      <td>73.48</td>\n",
       "      <td>56.34</td>\n",
       "      <td>41.33</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>no_word</th>\n",
       "      <td>53.88</td>\n",
       "      <td>61.12</td>\n",
       "      <td>42.74</td>\n",
       "      <td>27.14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"7\" valign=\"top\">char_lstm</th>\n",
       "      <th>ft_oov_tok</th>\n",
       "      <td>78.95</td>\n",
       "      <td>81.16</td>\n",
       "      <td>61.48</td>\n",
       "      <td>43.99</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_oov_yap</th>\n",
       "      <td>76.21</td>\n",
       "      <td>78.84</td>\n",
       "      <td>61.31</td>\n",
       "      <td>51.90</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_tok</th>\n",
       "      <td>74.61</td>\n",
       "      <td>80.40</td>\n",
       "      <td>62.44</td>\n",
       "      <td>48.45</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_yap</th>\n",
       "      <td>72.37</td>\n",
       "      <td>75.70</td>\n",
       "      <td>55.06</td>\n",
       "      <td>44.10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_tok</th>\n",
       "      <td>73.31</td>\n",
       "      <td>77.08</td>\n",
       "      <td>58.46</td>\n",
       "      <td>40.89</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_yap</th>\n",
       "      <td>69.22</td>\n",
       "      <td>73.32</td>\n",
       "      <td>55.07</td>\n",
       "      <td>40.52</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>no_word</th>\n",
       "      <td>54.46</td>\n",
       "      <td>60.96</td>\n",
       "      <td>45.17</td>\n",
       "      <td>26.66</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"7\" valign=\"top\">no_char</th>\n",
       "      <th>ft_oov_tok</th>\n",
       "      <td>79.91</td>\n",
       "      <td>81.45</td>\n",
       "      <td>60.08</td>\n",
       "      <td>46.34</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_oov_yap</th>\n",
       "      <td>74.97</td>\n",
       "      <td>77.85</td>\n",
       "      <td>56.14</td>\n",
       "      <td>46.41</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_tok</th>\n",
       "      <td>75.00</td>\n",
       "      <td>80.01</td>\n",
       "      <td>57.27</td>\n",
       "      <td>45.08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_yap</th>\n",
       "      <td>69.57</td>\n",
       "      <td>72.51</td>\n",
       "      <td>53.94</td>\n",
       "      <td>25.41</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_tok</th>\n",
       "      <td>70.26</td>\n",
       "      <td>76.33</td>\n",
       "      <td>55.93</td>\n",
       "      <td>33.70</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_yap</th>\n",
       "      <td>66.03</td>\n",
       "      <td>70.29</td>\n",
       "      <td>50.11</td>\n",
       "      <td>23.18</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>no_word</th>\n",
       "      <td>48.28</td>\n",
       "      <td>52.45</td>\n",
       "      <td>40.51</td>\n",
       "      <td>14.56</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"21\" valign=\"top\">token</th>\n",
       "      <th rowspan=\"7\" valign=\"top\">char_cnn</th>\n",
       "      <th>ft_oov_tok</th>\n",
       "      <td>79.75</td>\n",
       "      <td>82.34</td>\n",
       "      <td>62.61</td>\n",
       "      <td>43.86</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_oov_yap</th>\n",
       "      <td>77.09</td>\n",
       "      <td>78.27</td>\n",
       "      <td>66.14</td>\n",
       "      <td>50.51</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_tok</th>\n",
       "      <td>76.12</td>\n",
       "      <td>80.79</td>\n",
       "      <td>60.51</td>\n",
       "      <td>48.85</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_yap</th>\n",
       "      <td>70.77</td>\n",
       "      <td>75.41</td>\n",
       "      <td>55.75</td>\n",
       "      <td>40.57</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_tok</th>\n",
       "      <td>75.37</td>\n",
       "      <td>77.32</td>\n",
       "      <td>61.45</td>\n",
       "      <td>43.20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_yap</th>\n",
       "      <td>69.60</td>\n",
       "      <td>73.68</td>\n",
       "      <td>54.72</td>\n",
       "      <td>26.75</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>no_word</th>\n",
       "      <td>56.42</td>\n",
       "      <td>60.72</td>\n",
       "      <td>43.96</td>\n",
       "      <td>36.64</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"7\" valign=\"top\">char_lstm</th>\n",
       "      <th>ft_oov_tok</th>\n",
       "      <td>79.35</td>\n",
       "      <td>82.12</td>\n",
       "      <td>62.22</td>\n",
       "      <td>47.19</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_oov_yap</th>\n",
       "      <td>77.02</td>\n",
       "      <td>78.05</td>\n",
       "      <td>63.74</td>\n",
       "      <td>51.15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_tok</th>\n",
       "      <td>77.58</td>\n",
       "      <td>81.25</td>\n",
       "      <td>60.25</td>\n",
       "      <td>50.69</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_yap</th>\n",
       "      <td>71.90</td>\n",
       "      <td>74.35</td>\n",
       "      <td>55.26</td>\n",
       "      <td>44.40</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_tok</th>\n",
       "      <td>75.69</td>\n",
       "      <td>77.25</td>\n",
       "      <td>60.03</td>\n",
       "      <td>42.36</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_yap</th>\n",
       "      <td>70.24</td>\n",
       "      <td>73.91</td>\n",
       "      <td>55.51</td>\n",
       "      <td>23.05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>no_word</th>\n",
       "      <td>56.23</td>\n",
       "      <td>60.38</td>\n",
       "      <td>45.57</td>\n",
       "      <td>38.62</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"7\" valign=\"top\">no_char</th>\n",
       "      <th>ft_oov_tok</th>\n",
       "      <td>78.73</td>\n",
       "      <td>81.26</td>\n",
       "      <td>61.89</td>\n",
       "      <td>42.71</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_oov_yap</th>\n",
       "      <td>75.62</td>\n",
       "      <td>77.75</td>\n",
       "      <td>63.62</td>\n",
       "      <td>50.89</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_tok</th>\n",
       "      <td>74.43</td>\n",
       "      <td>80.77</td>\n",
       "      <td>58.38</td>\n",
       "      <td>47.64</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_yap</th>\n",
       "      <td>70.21</td>\n",
       "      <td>73.62</td>\n",
       "      <td>54.02</td>\n",
       "      <td>28.79</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_tok</th>\n",
       "      <td>70.94</td>\n",
       "      <td>76.49</td>\n",
       "      <td>59.17</td>\n",
       "      <td>37.26</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_yap</th>\n",
       "      <td>66.06</td>\n",
       "      <td>72.36</td>\n",
       "      <td>53.47</td>\n",
       "      <td>19.64</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>no_word</th>\n",
       "      <td>53.92</td>\n",
       "      <td>55.50</td>\n",
       "      <td>40.94</td>\n",
       "      <td>31.84</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "token_id_rep                                        1      2      3      4\n",
       "gold_name        unit     arch      w_embed                               \n",
       "morph_dev_gold   morph    char_cnn  ft_oov_tok  81.62  83.52  61.51  44.72\n",
       "                                    ft_oov_yap  81.71  83.81  61.97  51.07\n",
       "                                    ft_tok      80.98  83.42  61.36  48.60\n",
       "                                    ft_yap      80.02  83.43  61.79  48.80\n",
       "                                    glv_tok     80.26  81.46  59.17  41.30\n",
       "                                    glv_yap     80.14  81.09  60.94  42.80\n",
       "                                    no_word     61.72  69.73  49.43  38.58\n",
       "                          char_lstm ft_oov_tok  81.15  83.06  61.48  45.70\n",
       "                                    ft_oov_yap  82.21  83.06  59.19  44.97\n",
       "                                    ft_tok      80.37  82.91  60.72  47.90\n",
       "                                    ft_yap      80.47  82.96  58.64  48.54\n",
       "                                    glv_tok     80.09  81.40  60.01  40.98\n",
       "                                    glv_yap     80.42  81.39  64.35  47.10\n",
       "                                    no_word     62.47  69.47  44.22  35.83\n",
       "                          no_char   ft_oov_tok  81.44  83.02  62.86  48.97\n",
       "                                    ft_oov_yap  81.67  83.38  60.79  45.21\n",
       "                                    ft_tok      79.19  83.20  63.15  47.92\n",
       "                                    ft_yap      78.61  82.91  61.03  48.50\n",
       "                                    glv_tok     77.82  81.01  59.88  36.58\n",
       "                                    glv_yap     77.06  80.94  61.81  47.00\n",
       "                                    no_word     60.40  64.79  42.86  28.93\n",
       "morph_dev_pruned morph    char_cnn  ft_oov_tok  79.67  80.83  59.92  40.28\n",
       "                                    ft_oov_yap  79.93  82.01  59.15  47.67\n",
       "                                    ft_tok      77.16  81.09  58.62  43.03\n",
       "                                    ft_yap      77.96  81.56  58.88  45.89\n",
       "                                    glv_tok     77.50  78.61  56.60  38.09\n",
       "                                    glv_yap     78.14  78.97  58.96  41.93\n",
       "                                    no_word     57.28  66.14  46.98  34.34\n",
       "                          char_lstm ft_oov_tok  78.29  81.06  59.23  39.50\n",
       "                                    ft_oov_yap  80.20  82.04  56.81  43.09\n",
       "                                    ft_tok      76.40  80.26  58.77  43.64\n",
       "                                    ft_yap      78.18  81.13  56.55  45.73\n",
       "                                    glv_tok     76.40  79.38  55.76  40.74\n",
       "                                    glv_yap     78.06  79.21  62.96  41.23\n",
       "                                    no_word     58.56  65.64  40.59  35.13\n",
       "                          no_char   ft_oov_tok  78.49  81.65  62.66  40.49\n",
       "                                    ft_oov_yap  79.34  80.48  55.65  42.97\n",
       "                                    ft_tok      75.35  80.62  61.78  45.46\n",
       "                                    ft_yap      77.15  81.34  57.37  47.09\n",
       "                                    glv_tok     75.22  78.62  56.53  37.28\n",
       "                                    glv_yap     74.61  78.50  58.54  43.29\n",
       "                                    no_word     55.22  61.11  39.53  25.43\n",
       "morph_dev_yap    morph    char_cnn  ft_oov_tok  71.40  78.95  57.32  44.16\n",
       "                                    ft_oov_yap  71.88  78.99  56.54  48.04\n",
       "                                    ft_tok      70.63  78.23  55.63  47.61\n",
       "                                    ft_yap      69.94  78.03  55.34  46.17\n",
       "                                    glv_tok     70.52  75.43  53.24  40.58\n",
       "                                    glv_yap     70.44  74.87  53.39  40.70\n",
       "                                    no_word     53.41  65.76  43.54  39.05\n",
       "                          char_lstm ft_oov_tok  70.98  78.16  56.03  40.63\n",
       "                                    ft_oov_yap  72.23  78.26  54.53  41.53\n",
       "                                    ft_tok      70.07  77.21  55.27  44.71\n",
       "                                    ft_yap      69.90  77.22  52.80  44.29\n",
       "                                    glv_tok     69.98  75.40  54.01  42.34\n",
       "                                    glv_yap     70.22  75.55  57.13  44.74\n",
       "                                    no_word     54.28  66.32  39.28  33.29\n",
       "                          no_char   ft_oov_tok  70.91  78.15  58.40  44.89\n",
       "                                    ft_oov_yap  72.10  78.65  55.70  43.03\n",
       "                                    ft_tok      69.37  77.63  58.13  43.82\n",
       "                                    ft_yap      68.86  77.83  54.94  45.73\n",
       "                                    glv_tok     68.23  75.84  54.10  36.41\n",
       "                                    glv_yap     67.30  75.38  55.78  46.84\n",
       "                                    no_word     51.81  61.64  40.32  30.74\n",
       "token_dev        multitok char_cnn  ft_oov_tok  77.80  81.30  61.18  42.78\n",
       "                                    ft_oov_yap  76.27  77.80  60.99  47.07\n",
       "                                    ft_tok      74.46  80.64  60.44  42.78\n",
       "                                    ft_yap      71.22  75.84  55.58  42.15\n",
       "                                    glv_tok     72.69  77.42  59.08  39.86\n",
       "                                    glv_yap     70.50  73.48  56.34  41.33\n",
       "                                    no_word     53.88  61.12  42.74  27.14\n",
       "                          char_lstm ft_oov_tok  78.95  81.16  61.48  43.99\n",
       "                                    ft_oov_yap  76.21  78.84  61.31  51.90\n",
       "                                    ft_tok      74.61  80.40  62.44  48.45\n",
       "                                    ft_yap      72.37  75.70  55.06  44.10\n",
       "                                    glv_tok     73.31  77.08  58.46  40.89\n",
       "                                    glv_yap     69.22  73.32  55.07  40.52\n",
       "                                    no_word     54.46  60.96  45.17  26.66\n",
       "                          no_char   ft_oov_tok  79.91  81.45  60.08  46.34\n",
       "                                    ft_oov_yap  74.97  77.85  56.14  46.41\n",
       "                                    ft_tok      75.00  80.01  57.27  45.08\n",
       "                                    ft_yap      69.57  72.51  53.94  25.41\n",
       "                                    glv_tok     70.26  76.33  55.93  33.70\n",
       "                                    glv_yap     66.03  70.29  50.11  23.18\n",
       "                                    no_word     48.28  52.45  40.51  14.56\n",
       "                 token    char_cnn  ft_oov_tok  79.75  82.34  62.61  43.86\n",
       "                                    ft_oov_yap  77.09  78.27  66.14  50.51\n",
       "                                    ft_tok      76.12  80.79  60.51  48.85\n",
       "                                    ft_yap      70.77  75.41  55.75  40.57\n",
       "                                    glv_tok     75.37  77.32  61.45  43.20\n",
       "                                    glv_yap     69.60  73.68  54.72  26.75\n",
       "                                    no_word     56.42  60.72  43.96  36.64\n",
       "                          char_lstm ft_oov_tok  79.35  82.12  62.22  47.19\n",
       "                                    ft_oov_yap  77.02  78.05  63.74  51.15\n",
       "                                    ft_tok      77.58  81.25  60.25  50.69\n",
       "                                    ft_yap      71.90  74.35  55.26  44.40\n",
       "                                    glv_tok     75.69  77.25  60.03  42.36\n",
       "                                    glv_yap     70.24  73.91  55.51  23.05\n",
       "                                    no_word     56.23  60.38  45.57  38.62\n",
       "                          no_char   ft_oov_tok  78.73  81.26  61.89  42.71\n",
       "                                    ft_oov_yap  75.62  77.75  63.62  50.89\n",
       "                                    ft_tok      74.43  80.77  58.38  47.64\n",
       "                                    ft_yap      70.21  73.62  54.02  28.79\n",
       "                                    glv_tok     70.94  76.49  59.17  37.26\n",
       "                                    glv_yap     66.06  72.36  53.47  19.64\n",
       "                                    no_word     53.92  55.50  40.94  31.84"
      ]
     },
     "execution_count": 250,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gsc = group_scores(scores5, groupers=[ignore, ignore, ignore, lenset], ment_grouper=ignore, ment_cat_grouper=ignore)\n",
    "(gsc[gsc.gold_name.str.contains('dev')]\n",
    " .groupby(['gold_name', 'unit', 'arch', 'w_embed', 'token_id_rep'])\n",
    " .f.mean().mul(100).round(2).unstack())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## OOTV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 251,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>unk_type</th>\n",
       "      <th>()</th>\n",
       "      <th>(T,)</th>\n",
       "      <th>(T, TM)</th>\n",
       "      <th>(TM,)</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>gold_name</th>\n",
       "      <th>unit</th>\n",
       "      <th>arch</th>\n",
       "      <th>w_embed</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"21\" valign=\"top\">morph_dev_gold</th>\n",
       "      <th rowspan=\"21\" valign=\"top\">morph</th>\n",
       "      <th rowspan=\"7\" valign=\"top\">char_cnn</th>\n",
       "      <th>ft_oov_tok</th>\n",
       "      <td>0.822178</td>\n",
       "      <td>0.687515</td>\n",
       "      <td>0.533535</td>\n",
       "      <td>0.793864</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_oov_yap</th>\n",
       "      <td>0.830871</td>\n",
       "      <td>0.694968</td>\n",
       "      <td>0.457172</td>\n",
       "      <td>0.790695</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_tok</th>\n",
       "      <td>0.821035</td>\n",
       "      <td>0.681985</td>\n",
       "      <td>0.469091</td>\n",
       "      <td>0.789779</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_yap</th>\n",
       "      <td>0.832321</td>\n",
       "      <td>0.693843</td>\n",
       "      <td>0.418535</td>\n",
       "      <td>0.761516</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_tok</th>\n",
       "      <td>0.818826</td>\n",
       "      <td>0.659414</td>\n",
       "      <td>0.211555</td>\n",
       "      <td>0.766338</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_yap</th>\n",
       "      <td>0.820197</td>\n",
       "      <td>0.685386</td>\n",
       "      <td>0.342424</td>\n",
       "      <td>0.757847</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>no_word</th>\n",
       "      <td>0.787278</td>\n",
       "      <td>0.648385</td>\n",
       "      <td>-0.519596</td>\n",
       "      <td>0.350990</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"7\" valign=\"top\">char_lstm</th>\n",
       "      <th>ft_oov_tok</th>\n",
       "      <td>0.815305</td>\n",
       "      <td>0.686438</td>\n",
       "      <td>0.511717</td>\n",
       "      <td>0.795430</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_oov_yap</th>\n",
       "      <td>0.825925</td>\n",
       "      <td>0.696072</td>\n",
       "      <td>0.401616</td>\n",
       "      <td>0.790227</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_tok</th>\n",
       "      <td>0.817964</td>\n",
       "      <td>0.686331</td>\n",
       "      <td>0.475960</td>\n",
       "      <td>0.777748</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_yap</th>\n",
       "      <td>0.827873</td>\n",
       "      <td>0.690425</td>\n",
       "      <td>0.392020</td>\n",
       "      <td>0.764060</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_tok</th>\n",
       "      <td>0.811900</td>\n",
       "      <td>0.657389</td>\n",
       "      <td>0.372960</td>\n",
       "      <td>0.771990</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_yap</th>\n",
       "      <td>0.817194</td>\n",
       "      <td>0.691865</td>\n",
       "      <td>0.437778</td>\n",
       "      <td>0.774625</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>no_word</th>\n",
       "      <td>0.786730</td>\n",
       "      <td>0.639893</td>\n",
       "      <td>-0.557778</td>\n",
       "      <td>0.350393</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"7\" valign=\"top\">no_char</th>\n",
       "      <th>ft_oov_tok</th>\n",
       "      <td>0.829734</td>\n",
       "      <td>0.691140</td>\n",
       "      <td>0.534545</td>\n",
       "      <td>0.782692</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_oov_yap</th>\n",
       "      <td>0.834369</td>\n",
       "      <td>0.692069</td>\n",
       "      <td>0.434343</td>\n",
       "      <td>0.778655</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_tok</th>\n",
       "      <td>0.824097</td>\n",
       "      <td>0.690885</td>\n",
       "      <td>0.480497</td>\n",
       "      <td>0.762230</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_yap</th>\n",
       "      <td>0.836068</td>\n",
       "      <td>0.693894</td>\n",
       "      <td>0.375758</td>\n",
       "      <td>0.732187</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_tok</th>\n",
       "      <td>0.815581</td>\n",
       "      <td>0.648311</td>\n",
       "      <td>0.318135</td>\n",
       "      <td>0.732253</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_yap</th>\n",
       "      <td>0.816494</td>\n",
       "      <td>0.679197</td>\n",
       "      <td>0.487778</td>\n",
       "      <td>0.718766</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>no_word</th>\n",
       "      <td>0.791939</td>\n",
       "      <td>0.619117</td>\n",
       "      <td>-0.452778</td>\n",
       "      <td>0.199328</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"21\" valign=\"top\">morph_dev_pruned</th>\n",
       "      <th rowspan=\"21\" valign=\"top\">morph</th>\n",
       "      <th rowspan=\"7\" valign=\"top\">char_cnn</th>\n",
       "      <th>ft_oov_tok</th>\n",
       "      <td>0.805038</td>\n",
       "      <td>0.696114</td>\n",
       "      <td>0.364697</td>\n",
       "      <td>0.763052</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_oov_yap</th>\n",
       "      <td>0.813882</td>\n",
       "      <td>0.664788</td>\n",
       "      <td>0.457172</td>\n",
       "      <td>0.770258</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_tok</th>\n",
       "      <td>0.806279</td>\n",
       "      <td>0.588589</td>\n",
       "      <td>0.138333</td>\n",
       "      <td>0.752539</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_yap</th>\n",
       "      <td>0.815244</td>\n",
       "      <td>0.657342</td>\n",
       "      <td>0.273889</td>\n",
       "      <td>0.741341</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_tok</th>\n",
       "      <td>0.801752</td>\n",
       "      <td>0.611594</td>\n",
       "      <td>0.317203</td>\n",
       "      <td>0.727122</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_yap</th>\n",
       "      <td>0.809456</td>\n",
       "      <td>0.663865</td>\n",
       "      <td>0.226919</td>\n",
       "      <td>0.725131</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>no_word</th>\n",
       "      <td>0.756395</td>\n",
       "      <td>0.461964</td>\n",
       "      <td>-0.510556</td>\n",
       "      <td>0.330533</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"7\" valign=\"top\">char_lstm</th>\n",
       "      <th>ft_oov_tok</th>\n",
       "      <td>0.804639</td>\n",
       "      <td>0.621249</td>\n",
       "      <td>0.358333</td>\n",
       "      <td>0.760861</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_oov_yap</th>\n",
       "      <td>0.816085</td>\n",
       "      <td>0.647243</td>\n",
       "      <td>0.435556</td>\n",
       "      <td>0.770089</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_tok</th>\n",
       "      <td>0.801067</td>\n",
       "      <td>0.632550</td>\n",
       "      <td>0.303586</td>\n",
       "      <td>0.731483</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_yap</th>\n",
       "      <td>0.815245</td>\n",
       "      <td>0.654562</td>\n",
       "      <td>0.261111</td>\n",
       "      <td>0.737194</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_tok</th>\n",
       "      <td>0.796789</td>\n",
       "      <td>0.599711</td>\n",
       "      <td>0.393636</td>\n",
       "      <td>0.725134</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_yap</th>\n",
       "      <td>0.802280</td>\n",
       "      <td>0.648965</td>\n",
       "      <td>0.424927</td>\n",
       "      <td>0.746633</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>no_word</th>\n",
       "      <td>0.760139</td>\n",
       "      <td>0.499232</td>\n",
       "      <td>-0.657778</td>\n",
       "      <td>0.316903</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"7\" valign=\"top\">no_char</th>\n",
       "      <th>ft_oov_tok</th>\n",
       "      <td>0.811197</td>\n",
       "      <td>0.684377</td>\n",
       "      <td>0.351111</td>\n",
       "      <td>0.755512</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_oov_yap</th>\n",
       "      <td>0.812010</td>\n",
       "      <td>0.644579</td>\n",
       "      <td>0.235354</td>\n",
       "      <td>0.751582</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_tok</th>\n",
       "      <td>0.804631</td>\n",
       "      <td>0.615118</td>\n",
       "      <td>-0.045625</td>\n",
       "      <td>0.731914</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_yap</th>\n",
       "      <td>0.819558</td>\n",
       "      <td>0.644799</td>\n",
       "      <td>0.251515</td>\n",
       "      <td>0.721813</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_tok</th>\n",
       "      <td>0.804865</td>\n",
       "      <td>0.600740</td>\n",
       "      <td>0.205606</td>\n",
       "      <td>0.689444</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_yap</th>\n",
       "      <td>0.799961</td>\n",
       "      <td>0.633909</td>\n",
       "      <td>0.395253</td>\n",
       "      <td>0.686761</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>no_word</th>\n",
       "      <td>0.763380</td>\n",
       "      <td>0.342433</td>\n",
       "      <td>-0.700000</td>\n",
       "      <td>0.185191</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"21\" valign=\"top\">morph_dev_yap</th>\n",
       "      <th rowspan=\"21\" valign=\"top\">morph</th>\n",
       "      <th rowspan=\"7\" valign=\"top\">char_cnn</th>\n",
       "      <th>ft_oov_tok</th>\n",
       "      <td>0.753748</td>\n",
       "      <td>0.602180</td>\n",
       "      <td>0.452727</td>\n",
       "      <td>0.714417</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_oov_yap</th>\n",
       "      <td>0.758316</td>\n",
       "      <td>0.607201</td>\n",
       "      <td>0.451919</td>\n",
       "      <td>0.714039</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_tok</th>\n",
       "      <td>0.750643</td>\n",
       "      <td>0.585296</td>\n",
       "      <td>0.385820</td>\n",
       "      <td>0.706064</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_yap</th>\n",
       "      <td>0.760511</td>\n",
       "      <td>0.603091</td>\n",
       "      <td>0.383283</td>\n",
       "      <td>0.674773</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_tok</th>\n",
       "      <td>0.744839</td>\n",
       "      <td>0.564701</td>\n",
       "      <td>0.230553</td>\n",
       "      <td>0.682556</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_yap</th>\n",
       "      <td>0.744501</td>\n",
       "      <td>0.597494</td>\n",
       "      <td>0.196022</td>\n",
       "      <td>0.671523</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>no_word</th>\n",
       "      <td>0.712863</td>\n",
       "      <td>0.562143</td>\n",
       "      <td>-0.517374</td>\n",
       "      <td>0.312513</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"7\" valign=\"top\">char_lstm</th>\n",
       "      <th>ft_oov_tok</th>\n",
       "      <td>0.747125</td>\n",
       "      <td>0.602362</td>\n",
       "      <td>0.408889</td>\n",
       "      <td>0.708251</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_oov_yap</th>\n",
       "      <td>0.753730</td>\n",
       "      <td>0.608250</td>\n",
       "      <td>0.406061</td>\n",
       "      <td>0.712404</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_tok</th>\n",
       "      <td>0.745565</td>\n",
       "      <td>0.590927</td>\n",
       "      <td>0.367024</td>\n",
       "      <td>0.692798</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_yap</th>\n",
       "      <td>0.750877</td>\n",
       "      <td>0.596415</td>\n",
       "      <td>0.346869</td>\n",
       "      <td>0.676311</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_tok</th>\n",
       "      <td>0.737459</td>\n",
       "      <td>0.560213</td>\n",
       "      <td>0.353956</td>\n",
       "      <td>0.686935</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_yap</th>\n",
       "      <td>0.742670</td>\n",
       "      <td>0.601904</td>\n",
       "      <td>0.398430</td>\n",
       "      <td>0.684278</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>no_word</th>\n",
       "      <td>0.719516</td>\n",
       "      <td>0.550475</td>\n",
       "      <td>-0.661111</td>\n",
       "      <td>0.313294</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"7\" valign=\"top\">no_char</th>\n",
       "      <th>ft_oov_tok</th>\n",
       "      <td>0.758894</td>\n",
       "      <td>0.607067</td>\n",
       "      <td>0.422222</td>\n",
       "      <td>0.697720</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_oov_yap</th>\n",
       "      <td>0.762401</td>\n",
       "      <td>0.609822</td>\n",
       "      <td>0.404444</td>\n",
       "      <td>0.706724</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_tok</th>\n",
       "      <td>0.753776</td>\n",
       "      <td>0.599997</td>\n",
       "      <td>0.350163</td>\n",
       "      <td>0.681027</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_yap</th>\n",
       "      <td>0.763223</td>\n",
       "      <td>0.605705</td>\n",
       "      <td>0.312475</td>\n",
       "      <td>0.654892</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_tok</th>\n",
       "      <td>0.745827</td>\n",
       "      <td>0.566924</td>\n",
       "      <td>0.197424</td>\n",
       "      <td>0.650126</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_yap</th>\n",
       "      <td>0.747084</td>\n",
       "      <td>0.586978</td>\n",
       "      <td>0.446768</td>\n",
       "      <td>0.634460</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>no_word</th>\n",
       "      <td>0.721850</td>\n",
       "      <td>0.534444</td>\n",
       "      <td>-0.552778</td>\n",
       "      <td>0.170977</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "unk_type                                           ()      (T,)   (T, TM)  \\\n",
       "gold_name        unit  arch      w_embed                                    \n",
       "morph_dev_gold   morph char_cnn  ft_oov_tok  0.822178  0.687515  0.533535   \n",
       "                                 ft_oov_yap  0.830871  0.694968  0.457172   \n",
       "                                 ft_tok      0.821035  0.681985  0.469091   \n",
       "                                 ft_yap      0.832321  0.693843  0.418535   \n",
       "                                 glv_tok     0.818826  0.659414  0.211555   \n",
       "                                 glv_yap     0.820197  0.685386  0.342424   \n",
       "                                 no_word     0.787278  0.648385 -0.519596   \n",
       "                       char_lstm ft_oov_tok  0.815305  0.686438  0.511717   \n",
       "                                 ft_oov_yap  0.825925  0.696072  0.401616   \n",
       "                                 ft_tok      0.817964  0.686331  0.475960   \n",
       "                                 ft_yap      0.827873  0.690425  0.392020   \n",
       "                                 glv_tok     0.811900  0.657389  0.372960   \n",
       "                                 glv_yap     0.817194  0.691865  0.437778   \n",
       "                                 no_word     0.786730  0.639893 -0.557778   \n",
       "                       no_char   ft_oov_tok  0.829734  0.691140  0.534545   \n",
       "                                 ft_oov_yap  0.834369  0.692069  0.434343   \n",
       "                                 ft_tok      0.824097  0.690885  0.480497   \n",
       "                                 ft_yap      0.836068  0.693894  0.375758   \n",
       "                                 glv_tok     0.815581  0.648311  0.318135   \n",
       "                                 glv_yap     0.816494  0.679197  0.487778   \n",
       "                                 no_word     0.791939  0.619117 -0.452778   \n",
       "morph_dev_pruned morph char_cnn  ft_oov_tok  0.805038  0.696114  0.364697   \n",
       "                                 ft_oov_yap  0.813882  0.664788  0.457172   \n",
       "                                 ft_tok      0.806279  0.588589  0.138333   \n",
       "                                 ft_yap      0.815244  0.657342  0.273889   \n",
       "                                 glv_tok     0.801752  0.611594  0.317203   \n",
       "                                 glv_yap     0.809456  0.663865  0.226919   \n",
       "                                 no_word     0.756395  0.461964 -0.510556   \n",
       "                       char_lstm ft_oov_tok  0.804639  0.621249  0.358333   \n",
       "                                 ft_oov_yap  0.816085  0.647243  0.435556   \n",
       "                                 ft_tok      0.801067  0.632550  0.303586   \n",
       "                                 ft_yap      0.815245  0.654562  0.261111   \n",
       "                                 glv_tok     0.796789  0.599711  0.393636   \n",
       "                                 glv_yap     0.802280  0.648965  0.424927   \n",
       "                                 no_word     0.760139  0.499232 -0.657778   \n",
       "                       no_char   ft_oov_tok  0.811197  0.684377  0.351111   \n",
       "                                 ft_oov_yap  0.812010  0.644579  0.235354   \n",
       "                                 ft_tok      0.804631  0.615118 -0.045625   \n",
       "                                 ft_yap      0.819558  0.644799  0.251515   \n",
       "                                 glv_tok     0.804865  0.600740  0.205606   \n",
       "                                 glv_yap     0.799961  0.633909  0.395253   \n",
       "                                 no_word     0.763380  0.342433 -0.700000   \n",
       "morph_dev_yap    morph char_cnn  ft_oov_tok  0.753748  0.602180  0.452727   \n",
       "                                 ft_oov_yap  0.758316  0.607201  0.451919   \n",
       "                                 ft_tok      0.750643  0.585296  0.385820   \n",
       "                                 ft_yap      0.760511  0.603091  0.383283   \n",
       "                                 glv_tok     0.744839  0.564701  0.230553   \n",
       "                                 glv_yap     0.744501  0.597494  0.196022   \n",
       "                                 no_word     0.712863  0.562143 -0.517374   \n",
       "                       char_lstm ft_oov_tok  0.747125  0.602362  0.408889   \n",
       "                                 ft_oov_yap  0.753730  0.608250  0.406061   \n",
       "                                 ft_tok      0.745565  0.590927  0.367024   \n",
       "                                 ft_yap      0.750877  0.596415  0.346869   \n",
       "                                 glv_tok     0.737459  0.560213  0.353956   \n",
       "                                 glv_yap     0.742670  0.601904  0.398430   \n",
       "                                 no_word     0.719516  0.550475 -0.661111   \n",
       "                       no_char   ft_oov_tok  0.758894  0.607067  0.422222   \n",
       "                                 ft_oov_yap  0.762401  0.609822  0.404444   \n",
       "                                 ft_tok      0.753776  0.599997  0.350163   \n",
       "                                 ft_yap      0.763223  0.605705  0.312475   \n",
       "                                 glv_tok     0.745827  0.566924  0.197424   \n",
       "                                 glv_yap     0.747084  0.586978  0.446768   \n",
       "                                 no_word     0.721850  0.534444 -0.552778   \n",
       "\n",
       "unk_type                                        (TM,)  \n",
       "gold_name        unit  arch      w_embed               \n",
       "morph_dev_gold   morph char_cnn  ft_oov_tok  0.793864  \n",
       "                                 ft_oov_yap  0.790695  \n",
       "                                 ft_tok      0.789779  \n",
       "                                 ft_yap      0.761516  \n",
       "                                 glv_tok     0.766338  \n",
       "                                 glv_yap     0.757847  \n",
       "                                 no_word     0.350990  \n",
       "                       char_lstm ft_oov_tok  0.795430  \n",
       "                                 ft_oov_yap  0.790227  \n",
       "                                 ft_tok      0.777748  \n",
       "                                 ft_yap      0.764060  \n",
       "                                 glv_tok     0.771990  \n",
       "                                 glv_yap     0.774625  \n",
       "                                 no_word     0.350393  \n",
       "                       no_char   ft_oov_tok  0.782692  \n",
       "                                 ft_oov_yap  0.778655  \n",
       "                                 ft_tok      0.762230  \n",
       "                                 ft_yap      0.732187  \n",
       "                                 glv_tok     0.732253  \n",
       "                                 glv_yap     0.718766  \n",
       "                                 no_word     0.199328  \n",
       "morph_dev_pruned morph char_cnn  ft_oov_tok  0.763052  \n",
       "                                 ft_oov_yap  0.770258  \n",
       "                                 ft_tok      0.752539  \n",
       "                                 ft_yap      0.741341  \n",
       "                                 glv_tok     0.727122  \n",
       "                                 glv_yap     0.725131  \n",
       "                                 no_word     0.330533  \n",
       "                       char_lstm ft_oov_tok  0.760861  \n",
       "                                 ft_oov_yap  0.770089  \n",
       "                                 ft_tok      0.731483  \n",
       "                                 ft_yap      0.737194  \n",
       "                                 glv_tok     0.725134  \n",
       "                                 glv_yap     0.746633  \n",
       "                                 no_word     0.316903  \n",
       "                       no_char   ft_oov_tok  0.755512  \n",
       "                                 ft_oov_yap  0.751582  \n",
       "                                 ft_tok      0.731914  \n",
       "                                 ft_yap      0.721813  \n",
       "                                 glv_tok     0.689444  \n",
       "                                 glv_yap     0.686761  \n",
       "                                 no_word     0.185191  \n",
       "morph_dev_yap    morph char_cnn  ft_oov_tok  0.714417  \n",
       "                                 ft_oov_yap  0.714039  \n",
       "                                 ft_tok      0.706064  \n",
       "                                 ft_yap      0.674773  \n",
       "                                 glv_tok     0.682556  \n",
       "                                 glv_yap     0.671523  \n",
       "                                 no_word     0.312513  \n",
       "                       char_lstm ft_oov_tok  0.708251  \n",
       "                                 ft_oov_yap  0.712404  \n",
       "                                 ft_tok      0.692798  \n",
       "                                 ft_yap      0.676311  \n",
       "                                 glv_tok     0.686935  \n",
       "                                 glv_yap     0.684278  \n",
       "                                 no_word     0.313294  \n",
       "                       no_char   ft_oov_tok  0.697720  \n",
       "                                 ft_oov_yap  0.706724  \n",
       "                                 ft_tok      0.681027  \n",
       "                                 ft_yap      0.654892  \n",
       "                                 glv_tok     0.650126  \n",
       "                                 glv_yap     0.634460  \n",
       "                                 no_word     0.170977  "
      ]
     },
     "execution_count": 251,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gsc = group_scores(scores5, groupers=[ut_gm, ignore, ignore, ignore], ment_grouper=ignore, ment_cat_grouper=ignore)\n",
    "(gsc[gsc.gold_name.str.contains('morph_dev')]\n",
    " .groupby(['gold_name', 'unit', 'arch', 'w_embed', 'unk_type'])\n",
    " .f.mean().unstack())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 260,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>unk_type</th>\n",
       "      <th>()</th>\n",
       "      <th>(T,)</th>\n",
       "      <th>(T, TM)</th>\n",
       "      <th>(TM,)</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>gold_name</th>\n",
       "      <th>unit</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>morph_dev_gold</th>\n",
       "      <th>morph</th>\n",
       "      <td>83.09</td>\n",
       "      <td>69.50</td>\n",
       "      <td>45.72</td>\n",
       "      <td>79.07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>morph_dev_pruned</th>\n",
       "      <th>morph</th>\n",
       "      <td>81.39</td>\n",
       "      <td>66.48</td>\n",
       "      <td>45.72</td>\n",
       "      <td>77.03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">token_dev</th>\n",
       "      <th>multitok</th>\n",
       "      <td>80.06</td>\n",
       "      <td>63.40</td>\n",
       "      <td>32.36</td>\n",
       "      <td>78.23</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>token</th>\n",
       "      <td>81.64</td>\n",
       "      <td>64.21</td>\n",
       "      <td>26.87</td>\n",
       "      <td>77.67</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "unk_type                      ()   (T,)  (T, TM)  (TM,)\n",
       "gold_name        unit                                  \n",
       "morph_dev_gold   morph     83.09  69.50    45.72  79.07\n",
       "morph_dev_pruned morph     81.39  66.48    45.72  77.03\n",
       "token_dev        multitok  80.06  63.40    32.36  78.23\n",
       "                 token     81.64  64.21    26.87  77.67"
      ]
     },
     "execution_count": 260,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gsc = group_scores(scores5, groupers=[ut_gm, ignore, ignore, ignore], ment_grouper=ignore, ment_cat_grouper=ignore)\n",
    "(gsc[((gsc.gold_name.isin(['morph_dev_gold','morph_dev_pruned']))\n",
    "      & (gsc.arch=='char_cnn')\n",
    "      & (gsc.w_embed=='ft_oov_yap')\n",
    "\n",
    "     )\n",
    "    |\n",
    "     (\n",
    "        (gsc.gold_name.isin(['token_dev']))\n",
    "      & (gsc.arch=='char_lstm')\n",
    "      & (gsc.w_embed=='ft_oov_tok')\n",
    "\n",
    "     )]\n",
    " .groupby(['gold_name', 'unit', 'unk_type'])\n",
    " .f.mean().mul(100).round(2).unstack())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 267,
   "metadata": {},
   "outputs": [],
   "source": [
    "def unk_type_replace(t):\n",
    "    if t==('T',):\n",
    "        return 'Comp'\n",
    "    if t==():\n",
    "        return 'Known'\n",
    "    if t==('TM',):\n",
    "        return 'Real'\n",
    "    if t==('T', 'TM'):\n",
    "        return 'R+C'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 265,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index([(), ('T',), ('T', 'TM'), ('TM',)], dtype='object', name='unk_type')"
      ]
     },
     "execution_count": 265,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 274,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\\begin{tabular}{llllll}\n",
      "\\toprule\n",
      "          & \\textbf{unk\\_type} &          Known &           Real &           Comp &            R+C \\\\\n",
      "\\textbf{gold\\_name} & \\textbf{unit} &                &                &                &                \\\\\n",
      "\\midrule\n",
      "\\textbf{morph\\_dev\\_gold} & \\textbf{morph} &  $83.09 \\pm 0.9$ &  $79.07 \\pm 1.4$ &  $69.50 \\pm 1.3$ &  $45.72 \\pm 9.9$ \\\\\n",
      "\\textbf{morph\\_dev\\_pruned} & \\textbf{morph} &  $81.39 \\pm 1.0$ &  $77.03 \\pm 1.5$ &  $66.48 \\pm 1.2$ &  $45.72 \\pm 9.9$ \\\\\n",
      "\\textbf{token\\_dev} & \\textbf{multitok} &  $80.06 \\pm 1.0$ &  $78.23 \\pm 1.2$ &  $63.40 \\pm 2.6$ &  $32.36 \\pm 7.3$ \\\\\n",
      "          & \\textbf{token} &  $81.64 \\pm 0.9$ &  $77.67 \\pm 0.8$ &  $64.21 \\pm 2.5$ &  $26.87 \\pm 8.5$ \\\\\n",
      "\\bottomrule\n",
      "\\end{tabular}\n",
      "\n"
     ]
    }
   ],
   "source": [
    "gsc = (group_scores(scores5, groupers=[ut_gm, ignore, ignore, ignore], ment_grouper=ignore, ment_cat_grouper=ignore)\n",
    "      .assign(unk_type = lambda x: x.unk_type.apply(unk_type_replace)))\n",
    "x = (gsc[((gsc.gold_name.isin(['morph_dev_gold','morph_dev_pruned']))\n",
    "      & (gsc.arch=='char_cnn')\n",
    "      & (gsc.w_embed=='ft_oov_yap')\n",
    "\n",
    "     )\n",
    "    |\n",
    "     (\n",
    "        (gsc.gold_name.isin(['token_dev']))\n",
    "      & (gsc.arch=='char_lstm')\n",
    "      & (gsc.w_embed=='ft_oov_tok')\n",
    "\n",
    "     )]\n",
    " .groupby(['gold_name', 'unit', 'unk_type'])\n",
    " .f.agg(['mean', 'std']).mul(100).round(2)\n",
    "         .assign(mean = lambda x: '$'+x['mean'].apply('{:,.2f}'.format).astype(str)+' ± '+ x['std'].round(1).astype(str)+'$')[['mean']]\n",
    "         .unstack())\n",
    "x.columns = x.columns.droplevel(0)\n",
    "print (x[['Known', 'Real', 'Comp', 'R+C']].to_latex(bold_rows=True).replace('±', '\\pm').replace('\\$', '$'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 275,
   "metadata": {},
   "outputs": [],
   "source": [
    "from scipy.stats import ttest_ind"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 283,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Ttest_indResult(statistic=2.5674395676052857, pvalue=0.01937938613779738)"
      ]
     },
     "execution_count": 283,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tt = (gsc[(gsc.unk_type=='Comp') & ( ((gsc.gold_name=='morph_dev_pruned')\n",
    "      & (gsc.arch=='char_cnn')\n",
    "      & (gsc.w_embed=='ft_oov_yap')\n",
    "\n",
    "     )\n",
    "    |\n",
    "     (\n",
    "        (gsc.gold_name.isin(['token_dev']))\n",
    "      & (gsc.arch=='char_lstm')\n",
    "      & (gsc.w_embed=='ft_oov_tok')\n",
    "\n",
    "     ))]\n",
    ")\n",
    "\n",
    "ttest_ind(tt[tt.unit=='morph'].f, tt[tt.unit=='token'].f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 286,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Ttest_indResult(statistic=-1.9369654553486633, pvalue=0.06860642276609745)"
      ]
     },
     "execution_count": 286,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tt = (gsc[(gsc.unk_type=='Real') & ( ((gsc.gold_name=='morph_dev_pruned')\n",
    "      & (gsc.arch=='char_cnn')\n",
    "      & (gsc.w_embed=='ft_oov_yap')\n",
    "\n",
    "     )\n",
    "    |\n",
    "     (\n",
    "        (gsc.gold_name.isin(['token_dev']))\n",
    "      & (gsc.arch=='char_lstm')\n",
    "      & (gsc.w_embed=='ft_oov_tok')\n",
    "\n",
    "     ))]\n",
    ")\n",
    "\n",
    "ttest_ind(tt[tt.unit=='morph'].f, tt[tt.unit=='multitok'].f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 140,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>unk_type</th>\n",
       "      <th>()</th>\n",
       "      <th>(T,)</th>\n",
       "      <th>(T, TM)</th>\n",
       "      <th>(TM,)</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>gold_name</th>\n",
       "      <th>unit</th>\n",
       "      <th>arch</th>\n",
       "      <th>w_embed</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"42\" valign=\"top\">token_dev</th>\n",
       "      <th rowspan=\"21\" valign=\"top\">multitok</th>\n",
       "      <th rowspan=\"7\" valign=\"top\">char_cnn</th>\n",
       "      <th>ft_oov_tok</th>\n",
       "      <td>0.797595</td>\n",
       "      <td>0.630318</td>\n",
       "      <td>0.308345</td>\n",
       "      <td>0.771243</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_oov_yap</th>\n",
       "      <td>0.796952</td>\n",
       "      <td>0.597179</td>\n",
       "      <td>0.249293</td>\n",
       "      <td>0.727613</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_tok</th>\n",
       "      <td>0.800908</td>\n",
       "      <td>0.623261</td>\n",
       "      <td>0.142172</td>\n",
       "      <td>0.715171</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_yap</th>\n",
       "      <td>0.790428</td>\n",
       "      <td>0.457796</td>\n",
       "      <td>-1.000000</td>\n",
       "      <td>0.661115</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_tok</th>\n",
       "      <td>0.782409</td>\n",
       "      <td>0.544636</td>\n",
       "      <td>-0.046713</td>\n",
       "      <td>0.703852</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_yap</th>\n",
       "      <td>0.782695</td>\n",
       "      <td>0.340578</td>\n",
       "      <td>-0.875000</td>\n",
       "      <td>0.659396</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>no_word</th>\n",
       "      <td>0.754914</td>\n",
       "      <td>0.086791</td>\n",
       "      <td>-0.700000</td>\n",
       "      <td>0.285574</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"7\" valign=\"top\">char_lstm</th>\n",
       "      <th>ft_oov_tok</th>\n",
       "      <td>0.800648</td>\n",
       "      <td>0.633958</td>\n",
       "      <td>0.323636</td>\n",
       "      <td>0.782296</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_oov_yap</th>\n",
       "      <td>0.799922</td>\n",
       "      <td>0.593703</td>\n",
       "      <td>0.291515</td>\n",
       "      <td>0.736229</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_tok</th>\n",
       "      <td>0.801149</td>\n",
       "      <td>0.622028</td>\n",
       "      <td>0.028030</td>\n",
       "      <td>0.722791</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_yap</th>\n",
       "      <td>0.790918</td>\n",
       "      <td>0.477419</td>\n",
       "      <td>-1.000000</td>\n",
       "      <td>0.668950</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_tok</th>\n",
       "      <td>0.789994</td>\n",
       "      <td>0.554734</td>\n",
       "      <td>-0.271708</td>\n",
       "      <td>0.697833</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_yap</th>\n",
       "      <td>0.777500</td>\n",
       "      <td>0.361928</td>\n",
       "      <td>-0.880000</td>\n",
       "      <td>0.640901</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>no_word</th>\n",
       "      <td>0.755416</td>\n",
       "      <td>0.160329</td>\n",
       "      <td>-0.800000</td>\n",
       "      <td>0.291716</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"7\" valign=\"top\">no_char</th>\n",
       "      <th>ft_oov_tok</th>\n",
       "      <td>0.799358</td>\n",
       "      <td>0.660507</td>\n",
       "      <td>0.169417</td>\n",
       "      <td>0.795378</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_oov_yap</th>\n",
       "      <td>0.779761</td>\n",
       "      <td>0.594731</td>\n",
       "      <td>0.012727</td>\n",
       "      <td>0.727356</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_tok</th>\n",
       "      <td>0.798358</td>\n",
       "      <td>0.628164</td>\n",
       "      <td>-0.171888</td>\n",
       "      <td>0.721140</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_yap</th>\n",
       "      <td>0.780837</td>\n",
       "      <td>0.442400</td>\n",
       "      <td>-1.000000</td>\n",
       "      <td>0.608777</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_tok</th>\n",
       "      <td>0.781651</td>\n",
       "      <td>0.531278</td>\n",
       "      <td>-0.761818</td>\n",
       "      <td>0.656692</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_yap</th>\n",
       "      <td>0.767949</td>\n",
       "      <td>0.324679</td>\n",
       "      <td>-0.880000</td>\n",
       "      <td>0.567150</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>no_word</th>\n",
       "      <td>0.734351</td>\n",
       "      <td>0.099745</td>\n",
       "      <td>-1.000000</td>\n",
       "      <td>0.202486</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"21\" valign=\"top\">token</th>\n",
       "      <th rowspan=\"7\" valign=\"top\">char_cnn</th>\n",
       "      <th>ft_oov_tok</th>\n",
       "      <td>0.817111</td>\n",
       "      <td>0.652524</td>\n",
       "      <td>0.306970</td>\n",
       "      <td>0.780089</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_oov_yap</th>\n",
       "      <td>0.805971</td>\n",
       "      <td>0.584149</td>\n",
       "      <td>0.109798</td>\n",
       "      <td>0.745051</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_tok</th>\n",
       "      <td>0.810920</td>\n",
       "      <td>0.641690</td>\n",
       "      <td>-0.397778</td>\n",
       "      <td>0.726085</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_yap</th>\n",
       "      <td>0.795888</td>\n",
       "      <td>0.474744</td>\n",
       "      <td>-1.000000</td>\n",
       "      <td>0.635088</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_tok</th>\n",
       "      <td>0.792902</td>\n",
       "      <td>0.531939</td>\n",
       "      <td>-0.530303</td>\n",
       "      <td>0.732236</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_yap</th>\n",
       "      <td>0.770903</td>\n",
       "      <td>0.369180</td>\n",
       "      <td>-0.760000</td>\n",
       "      <td>0.647498</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>no_word</th>\n",
       "      <td>0.773292</td>\n",
       "      <td>0.136044</td>\n",
       "      <td>-1.000000</td>\n",
       "      <td>0.262716</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"7\" valign=\"top\">char_lstm</th>\n",
       "      <th>ft_oov_tok</th>\n",
       "      <td>0.816433</td>\n",
       "      <td>0.642129</td>\n",
       "      <td>0.268687</td>\n",
       "      <td>0.776677</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_oov_yap</th>\n",
       "      <td>0.799790</td>\n",
       "      <td>0.581078</td>\n",
       "      <td>0.353636</td>\n",
       "      <td>0.745659</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_tok</th>\n",
       "      <td>0.813877</td>\n",
       "      <td>0.644188</td>\n",
       "      <td>-0.259596</td>\n",
       "      <td>0.746869</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_yap</th>\n",
       "      <td>0.791385</td>\n",
       "      <td>0.468675</td>\n",
       "      <td>-0.880000</td>\n",
       "      <td>0.649295</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_tok</th>\n",
       "      <td>0.795105</td>\n",
       "      <td>0.544663</td>\n",
       "      <td>0.067980</td>\n",
       "      <td>0.725099</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_yap</th>\n",
       "      <td>0.776317</td>\n",
       "      <td>0.367429</td>\n",
       "      <td>-0.637778</td>\n",
       "      <td>0.651485</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>no_word</th>\n",
       "      <td>0.756457</td>\n",
       "      <td>0.102009</td>\n",
       "      <td>-0.900000</td>\n",
       "      <td>0.301990</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"7\" valign=\"top\">no_char</th>\n",
       "      <th>ft_oov_tok</th>\n",
       "      <td>0.813061</td>\n",
       "      <td>0.608385</td>\n",
       "      <td>0.313333</td>\n",
       "      <td>0.767210</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_oov_yap</th>\n",
       "      <td>0.805618</td>\n",
       "      <td>0.521275</td>\n",
       "      <td>0.321414</td>\n",
       "      <td>0.726865</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_tok</th>\n",
       "      <td>0.811931</td>\n",
       "      <td>0.597331</td>\n",
       "      <td>-0.513333</td>\n",
       "      <td>0.704907</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_yap</th>\n",
       "      <td>0.793418</td>\n",
       "      <td>0.416617</td>\n",
       "      <td>-1.000000</td>\n",
       "      <td>0.610300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_tok</th>\n",
       "      <td>0.792238</td>\n",
       "      <td>0.473870</td>\n",
       "      <td>-0.881818</td>\n",
       "      <td>0.667783</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_yap</th>\n",
       "      <td>0.769771</td>\n",
       "      <td>0.340744</td>\n",
       "      <td>-0.875000</td>\n",
       "      <td>0.578671</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>no_word</th>\n",
       "      <td>0.744116</td>\n",
       "      <td>0.090221</td>\n",
       "      <td>-0.200000</td>\n",
       "      <td>0.178180</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "unk_type                                       ()      (T,)   (T, TM)  \\\n",
       "gold_name unit     arch      w_embed                                    \n",
       "token_dev multitok char_cnn  ft_oov_tok  0.797595  0.630318  0.308345   \n",
       "                             ft_oov_yap  0.796952  0.597179  0.249293   \n",
       "                             ft_tok      0.800908  0.623261  0.142172   \n",
       "                             ft_yap      0.790428  0.457796 -1.000000   \n",
       "                             glv_tok     0.782409  0.544636 -0.046713   \n",
       "                             glv_yap     0.782695  0.340578 -0.875000   \n",
       "                             no_word     0.754914  0.086791 -0.700000   \n",
       "                   char_lstm ft_oov_tok  0.800648  0.633958  0.323636   \n",
       "                             ft_oov_yap  0.799922  0.593703  0.291515   \n",
       "                             ft_tok      0.801149  0.622028  0.028030   \n",
       "                             ft_yap      0.790918  0.477419 -1.000000   \n",
       "                             glv_tok     0.789994  0.554734 -0.271708   \n",
       "                             glv_yap     0.777500  0.361928 -0.880000   \n",
       "                             no_word     0.755416  0.160329 -0.800000   \n",
       "                   no_char   ft_oov_tok  0.799358  0.660507  0.169417   \n",
       "                             ft_oov_yap  0.779761  0.594731  0.012727   \n",
       "                             ft_tok      0.798358  0.628164 -0.171888   \n",
       "                             ft_yap      0.780837  0.442400 -1.000000   \n",
       "                             glv_tok     0.781651  0.531278 -0.761818   \n",
       "                             glv_yap     0.767949  0.324679 -0.880000   \n",
       "                             no_word     0.734351  0.099745 -1.000000   \n",
       "          token    char_cnn  ft_oov_tok  0.817111  0.652524  0.306970   \n",
       "                             ft_oov_yap  0.805971  0.584149  0.109798   \n",
       "                             ft_tok      0.810920  0.641690 -0.397778   \n",
       "                             ft_yap      0.795888  0.474744 -1.000000   \n",
       "                             glv_tok     0.792902  0.531939 -0.530303   \n",
       "                             glv_yap     0.770903  0.369180 -0.760000   \n",
       "                             no_word     0.773292  0.136044 -1.000000   \n",
       "                   char_lstm ft_oov_tok  0.816433  0.642129  0.268687   \n",
       "                             ft_oov_yap  0.799790  0.581078  0.353636   \n",
       "                             ft_tok      0.813877  0.644188 -0.259596   \n",
       "                             ft_yap      0.791385  0.468675 -0.880000   \n",
       "                             glv_tok     0.795105  0.544663  0.067980   \n",
       "                             glv_yap     0.776317  0.367429 -0.637778   \n",
       "                             no_word     0.756457  0.102009 -0.900000   \n",
       "                   no_char   ft_oov_tok  0.813061  0.608385  0.313333   \n",
       "                             ft_oov_yap  0.805618  0.521275  0.321414   \n",
       "                             ft_tok      0.811931  0.597331 -0.513333   \n",
       "                             ft_yap      0.793418  0.416617 -1.000000   \n",
       "                             glv_tok     0.792238  0.473870 -0.881818   \n",
       "                             glv_yap     0.769771  0.340744 -0.875000   \n",
       "                             no_word     0.744116  0.090221 -0.200000   \n",
       "\n",
       "unk_type                                    (TM,)  \n",
       "gold_name unit     arch      w_embed               \n",
       "token_dev multitok char_cnn  ft_oov_tok  0.771243  \n",
       "                             ft_oov_yap  0.727613  \n",
       "                             ft_tok      0.715171  \n",
       "                             ft_yap      0.661115  \n",
       "                             glv_tok     0.703852  \n",
       "                             glv_yap     0.659396  \n",
       "                             no_word     0.285574  \n",
       "                   char_lstm ft_oov_tok  0.782296  \n",
       "                             ft_oov_yap  0.736229  \n",
       "                             ft_tok      0.722791  \n",
       "                             ft_yap      0.668950  \n",
       "                             glv_tok     0.697833  \n",
       "                             glv_yap     0.640901  \n",
       "                             no_word     0.291716  \n",
       "                   no_char   ft_oov_tok  0.795378  \n",
       "                             ft_oov_yap  0.727356  \n",
       "                             ft_tok      0.721140  \n",
       "                             ft_yap      0.608777  \n",
       "                             glv_tok     0.656692  \n",
       "                             glv_yap     0.567150  \n",
       "                             no_word     0.202486  \n",
       "          token    char_cnn  ft_oov_tok  0.780089  \n",
       "                             ft_oov_yap  0.745051  \n",
       "                             ft_tok      0.726085  \n",
       "                             ft_yap      0.635088  \n",
       "                             glv_tok     0.732236  \n",
       "                             glv_yap     0.647498  \n",
       "                             no_word     0.262716  \n",
       "                   char_lstm ft_oov_tok  0.776677  \n",
       "                             ft_oov_yap  0.745659  \n",
       "                             ft_tok      0.746869  \n",
       "                             ft_yap      0.649295  \n",
       "                             glv_tok     0.725099  \n",
       "                             glv_yap     0.651485  \n",
       "                             no_word     0.301990  \n",
       "                   no_char   ft_oov_tok  0.767210  \n",
       "                             ft_oov_yap  0.726865  \n",
       "                             ft_tok      0.704907  \n",
       "                             ft_yap      0.610300  \n",
       "                             glv_tok     0.667783  \n",
       "                             glv_yap     0.578671  \n",
       "                             no_word     0.178180  "
      ]
     },
     "execution_count": 140,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gsc = group_scores(scores4, groupers=[ut_gm, ignore, ignore], ment_grouper=ignore, ment_cat_grouper=ignore)\n",
    "(gsc[gsc.gold_name.str.contains('token_dev')]\n",
    " .groupby(['gold_name', 'unit', 'arch', 'w_embed', 'unk_type'])\n",
    " .f.mean().unstack())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Maybe T has more ORGs, and that's why it's harder?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 141,
   "metadata": {},
   "outputs": [],
   "source": [
    "pd.set_option(\"max_rows\", 300)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 142,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>unk_type</th>\n",
       "      <th>()</th>\n",
       "      <th>(T,)</th>\n",
       "      <th>(T, TM)</th>\n",
       "      <th>(TM,)</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ment_cat</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>ANG</th>\n",
       "      <td>3.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>DUC</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>EVE</th>\n",
       "      <td>5.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>FAC</th>\n",
       "      <td>5.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>GPE</th>\n",
       "      <td>76.0</td>\n",
       "      <td>15.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>30.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>LOC</th>\n",
       "      <td>20.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>6.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ORG</th>\n",
       "      <td>61.0</td>\n",
       "      <td>11.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>43.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>PER</th>\n",
       "      <td>70.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>119.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>WOA</th>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "unk_type    ()  (T,)  (T, TM)  (TM,)\n",
       "ment_cat                            \n",
       "ANG        3.0   0.0      NaN    0.0\n",
       "DUC        1.0   0.0      0.0    1.0\n",
       "EVE        5.0   5.0      2.0    0.0\n",
       "FAC        5.0   5.0      0.0    2.0\n",
       "GPE       76.0  15.0      NaN   30.0\n",
       "LOC       20.0   2.0      0.0    6.0\n",
       "ORG       61.0  11.0      4.0   43.0\n",
       "PER       70.0   4.0      0.0  119.0\n",
       "WOA        3.0   3.0      0.0    3.0"
      ]
     },
     "execution_count": 142,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gsc = group_scores(scores4, groupers=[ut_gm, ignore, ignore], \n",
    "                   ment_grouper=ignore, ment_cat_grouper=same)\n",
    "xx = (gsc[gsc.gold_name.str.contains('morph_dev')]\n",
    " .groupby(['ment_cat', 'unk_type'])\n",
    " .true.mean().unstack())\n",
    "xx"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 143,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>unk_type</th>\n",
       "      <th>()</th>\n",
       "      <th>(T,)</th>\n",
       "      <th>(T, TM)</th>\n",
       "      <th>(TM,)</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ment_cat</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>ANG</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>DUC</th>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.500000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>EVE</th>\n",
       "      <td>0.416667</td>\n",
       "      <td>0.416667</td>\n",
       "      <td>0.166667</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>FAC</th>\n",
       "      <td>0.416667</td>\n",
       "      <td>0.416667</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.166667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>GPE</th>\n",
       "      <td>0.628099</td>\n",
       "      <td>0.123967</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.247934</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>LOC</th>\n",
       "      <td>0.714286</td>\n",
       "      <td>0.071429</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.214286</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ORG</th>\n",
       "      <td>0.512605</td>\n",
       "      <td>0.092437</td>\n",
       "      <td>0.033613</td>\n",
       "      <td>0.361345</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>PER</th>\n",
       "      <td>0.362694</td>\n",
       "      <td>0.020725</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.616580</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>WOA</th>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.333333</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "unk_type        ()      (T,)   (T, TM)     (TM,)\n",
       "ment_cat                                        \n",
       "ANG       1.000000  0.000000       NaN  0.000000\n",
       "DUC       0.500000  0.000000  0.000000  0.500000\n",
       "EVE       0.416667  0.416667  0.166667  0.000000\n",
       "FAC       0.416667  0.416667  0.000000  0.166667\n",
       "GPE       0.628099  0.123967       NaN  0.247934\n",
       "LOC       0.714286  0.071429  0.000000  0.214286\n",
       "ORG       0.512605  0.092437  0.033613  0.361345\n",
       "PER       0.362694  0.020725  0.000000  0.616580\n",
       "WOA       0.333333  0.333333  0.000000  0.333333"
      ]
     },
     "execution_count": 143,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(xx.T / xx.T.sum()).T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 144,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>unk_type</th>\n",
       "      <th>()</th>\n",
       "      <th>(T,)</th>\n",
       "      <th>(T, TM)</th>\n",
       "      <th>(TM,)</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>gold_name</th>\n",
       "      <th>unit</th>\n",
       "      <th>arch</th>\n",
       "      <th>w_embed</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"21\" valign=\"top\">morph_dev_gold</th>\n",
       "      <th rowspan=\"21\" valign=\"top\">morph</th>\n",
       "      <th rowspan=\"7\" valign=\"top\">char_cnn</th>\n",
       "      <th>ft_oov_tok</th>\n",
       "      <td>0.754281</td>\n",
       "      <td>0.677696</td>\n",
       "      <td>0.732143</td>\n",
       "      <td>0.530702</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_oov_yap</th>\n",
       "      <td>0.753601</td>\n",
       "      <td>0.705882</td>\n",
       "      <td>0.628571</td>\n",
       "      <td>0.553281</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_tok</th>\n",
       "      <td>0.757968</td>\n",
       "      <td>0.696315</td>\n",
       "      <td>0.627778</td>\n",
       "      <td>0.530970</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_yap</th>\n",
       "      <td>0.759375</td>\n",
       "      <td>0.713072</td>\n",
       "      <td>0.594048</td>\n",
       "      <td>0.534189</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_tok</th>\n",
       "      <td>0.737404</td>\n",
       "      <td>0.650245</td>\n",
       "      <td>0.422619</td>\n",
       "      <td>0.426602</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_yap</th>\n",
       "      <td>0.751105</td>\n",
       "      <td>0.683893</td>\n",
       "      <td>0.551429</td>\n",
       "      <td>0.450434</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>no_word</th>\n",
       "      <td>0.696903</td>\n",
       "      <td>0.595184</td>\n",
       "      <td>-0.389286</td>\n",
       "      <td>0.226856</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"7\" valign=\"top\">char_lstm</th>\n",
       "      <th>ft_oov_tok</th>\n",
       "      <td>0.757239</td>\n",
       "      <td>0.692565</td>\n",
       "      <td>0.696429</td>\n",
       "      <td>0.544201</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_oov_yap</th>\n",
       "      <td>0.754131</td>\n",
       "      <td>0.713072</td>\n",
       "      <td>0.571429</td>\n",
       "      <td>0.545209</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_tok</th>\n",
       "      <td>0.757545</td>\n",
       "      <td>0.700178</td>\n",
       "      <td>0.635714</td>\n",
       "      <td>0.520620</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_yap</th>\n",
       "      <td>0.754693</td>\n",
       "      <td>0.709150</td>\n",
       "      <td>0.559524</td>\n",
       "      <td>0.562202</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_tok</th>\n",
       "      <td>0.733897</td>\n",
       "      <td>0.682190</td>\n",
       "      <td>0.608333</td>\n",
       "      <td>0.433548</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_yap</th>\n",
       "      <td>0.746523</td>\n",
       "      <td>0.710997</td>\n",
       "      <td>0.661905</td>\n",
       "      <td>0.478477</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>no_word</th>\n",
       "      <td>0.674480</td>\n",
       "      <td>0.525507</td>\n",
       "      <td>-0.433333</td>\n",
       "      <td>0.214480</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"7\" valign=\"top\">no_char</th>\n",
       "      <th>ft_oov_tok</th>\n",
       "      <td>0.763919</td>\n",
       "      <td>0.698039</td>\n",
       "      <td>0.700000</td>\n",
       "      <td>0.509599</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_oov_yap</th>\n",
       "      <td>0.760657</td>\n",
       "      <td>0.713072</td>\n",
       "      <td>0.607143</td>\n",
       "      <td>0.530753</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_tok</th>\n",
       "      <td>0.767361</td>\n",
       "      <td>0.701961</td>\n",
       "      <td>0.621429</td>\n",
       "      <td>0.534548</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_yap</th>\n",
       "      <td>0.766936</td>\n",
       "      <td>0.705882</td>\n",
       "      <td>0.535714</td>\n",
       "      <td>0.545916</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_tok</th>\n",
       "      <td>0.733222</td>\n",
       "      <td>0.639134</td>\n",
       "      <td>0.528571</td>\n",
       "      <td>0.442740</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_yap</th>\n",
       "      <td>0.746239</td>\n",
       "      <td>0.686144</td>\n",
       "      <td>0.700000</td>\n",
       "      <td>0.458695</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>no_word</th>\n",
       "      <td>0.688105</td>\n",
       "      <td>0.484172</td>\n",
       "      <td>-0.233333</td>\n",
       "      <td>0.140628</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"21\" valign=\"top\">morph_dev_pruned</th>\n",
       "      <th rowspan=\"21\" valign=\"top\">morph</th>\n",
       "      <th rowspan=\"7\" valign=\"top\">char_cnn</th>\n",
       "      <th>ft_oov_tok</th>\n",
       "      <td>0.733686</td>\n",
       "      <td>0.673775</td>\n",
       "      <td>0.502381</td>\n",
       "      <td>0.456523</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_oov_yap</th>\n",
       "      <td>0.735008</td>\n",
       "      <td>0.694118</td>\n",
       "      <td>0.628571</td>\n",
       "      <td>0.501492</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_tok</th>\n",
       "      <td>0.739054</td>\n",
       "      <td>0.681527</td>\n",
       "      <td>0.248810</td>\n",
       "      <td>0.497062</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_yap</th>\n",
       "      <td>0.726999</td>\n",
       "      <td>0.705229</td>\n",
       "      <td>0.373810</td>\n",
       "      <td>0.517877</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_tok</th>\n",
       "      <td>0.724567</td>\n",
       "      <td>0.627292</td>\n",
       "      <td>0.510714</td>\n",
       "      <td>0.359413</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_yap</th>\n",
       "      <td>0.735896</td>\n",
       "      <td>0.676539</td>\n",
       "      <td>0.389524</td>\n",
       "      <td>0.376554</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>no_word</th>\n",
       "      <td>0.674766</td>\n",
       "      <td>0.572472</td>\n",
       "      <td>-0.380952</td>\n",
       "      <td>0.171393</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"7\" valign=\"top\">char_lstm</th>\n",
       "      <th>ft_oov_tok</th>\n",
       "      <td>0.740776</td>\n",
       "      <td>0.615833</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.463889</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_oov_yap</th>\n",
       "      <td>0.742663</td>\n",
       "      <td>0.709150</td>\n",
       "      <td>0.571429</td>\n",
       "      <td>0.521660</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_tok</th>\n",
       "      <td>0.721855</td>\n",
       "      <td>0.688368</td>\n",
       "      <td>0.423810</td>\n",
       "      <td>0.468025</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_yap</th>\n",
       "      <td>0.741256</td>\n",
       "      <td>0.629412</td>\n",
       "      <td>0.354762</td>\n",
       "      <td>0.459583</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_tok</th>\n",
       "      <td>0.720376</td>\n",
       "      <td>0.653655</td>\n",
       "      <td>0.608333</td>\n",
       "      <td>0.361926</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_yap</th>\n",
       "      <td>0.730325</td>\n",
       "      <td>0.706422</td>\n",
       "      <td>0.666667</td>\n",
       "      <td>0.418184</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>no_word</th>\n",
       "      <td>0.654506</td>\n",
       "      <td>0.525817</td>\n",
       "      <td>-0.433333</td>\n",
       "      <td>0.117691</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"7\" valign=\"top\">no_char</th>\n",
       "      <th>ft_oov_tok</th>\n",
       "      <td>0.744478</td>\n",
       "      <td>0.701961</td>\n",
       "      <td>0.471429</td>\n",
       "      <td>0.440417</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_oov_yap</th>\n",
       "      <td>0.733918</td>\n",
       "      <td>0.618137</td>\n",
       "      <td>0.328571</td>\n",
       "      <td>0.501661</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_tok</th>\n",
       "      <td>0.743230</td>\n",
       "      <td>0.621324</td>\n",
       "      <td>0.048810</td>\n",
       "      <td>0.508105</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_yap</th>\n",
       "      <td>0.752647</td>\n",
       "      <td>0.610703</td>\n",
       "      <td>0.376190</td>\n",
       "      <td>0.545437</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_tok</th>\n",
       "      <td>0.727446</td>\n",
       "      <td>0.560392</td>\n",
       "      <td>0.340476</td>\n",
       "      <td>0.360642</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_yap</th>\n",
       "      <td>0.742650</td>\n",
       "      <td>0.655394</td>\n",
       "      <td>0.566667</td>\n",
       "      <td>0.391374</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>no_word</th>\n",
       "      <td>0.661586</td>\n",
       "      <td>0.513313</td>\n",
       "      <td>-0.400000</td>\n",
       "      <td>0.120680</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"21\" valign=\"top\">morph_dev_yap</th>\n",
       "      <th rowspan=\"21\" valign=\"top\">morph</th>\n",
       "      <th rowspan=\"7\" valign=\"top\">char_cnn</th>\n",
       "      <th>ft_oov_tok</th>\n",
       "      <td>0.716405</td>\n",
       "      <td>0.593824</td>\n",
       "      <td>0.628571</td>\n",
       "      <td>0.458156</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_oov_yap</th>\n",
       "      <td>0.711427</td>\n",
       "      <td>0.613971</td>\n",
       "      <td>0.621429</td>\n",
       "      <td>0.490028</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_tok</th>\n",
       "      <td>0.709761</td>\n",
       "      <td>0.601618</td>\n",
       "      <td>0.577778</td>\n",
       "      <td>0.462495</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_yap</th>\n",
       "      <td>0.711741</td>\n",
       "      <td>0.629412</td>\n",
       "      <td>0.552381</td>\n",
       "      <td>0.463550</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_tok</th>\n",
       "      <td>0.685922</td>\n",
       "      <td>0.546046</td>\n",
       "      <td>0.422619</td>\n",
       "      <td>0.355605</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_yap</th>\n",
       "      <td>0.702825</td>\n",
       "      <td>0.599232</td>\n",
       "      <td>0.389524</td>\n",
       "      <td>0.374200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>no_word</th>\n",
       "      <td>0.638431</td>\n",
       "      <td>0.519281</td>\n",
       "      <td>-0.285714</td>\n",
       "      <td>0.192721</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"7\" valign=\"top\">char_lstm</th>\n",
       "      <th>ft_oov_tok</th>\n",
       "      <td>0.716419</td>\n",
       "      <td>0.615833</td>\n",
       "      <td>0.571429</td>\n",
       "      <td>0.447852</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_oov_yap</th>\n",
       "      <td>0.713452</td>\n",
       "      <td>0.633088</td>\n",
       "      <td>0.571429</td>\n",
       "      <td>0.483582</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_tok</th>\n",
       "      <td>0.703021</td>\n",
       "      <td>0.611471</td>\n",
       "      <td>0.532143</td>\n",
       "      <td>0.444307</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_yap</th>\n",
       "      <td>0.701572</td>\n",
       "      <td>0.629412</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.478954</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_tok</th>\n",
       "      <td>0.681397</td>\n",
       "      <td>0.579869</td>\n",
       "      <td>0.581667</td>\n",
       "      <td>0.368878</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_yap</th>\n",
       "      <td>0.701187</td>\n",
       "      <td>0.645899</td>\n",
       "      <td>0.666667</td>\n",
       "      <td>0.399180</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>no_word</th>\n",
       "      <td>0.626685</td>\n",
       "      <td>0.436242</td>\n",
       "      <td>-0.433333</td>\n",
       "      <td>0.141314</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"7\" valign=\"top\">no_char</th>\n",
       "      <th>ft_oov_tok</th>\n",
       "      <td>0.720240</td>\n",
       "      <td>0.617647</td>\n",
       "      <td>0.564286</td>\n",
       "      <td>0.421550</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_oov_yap</th>\n",
       "      <td>0.720024</td>\n",
       "      <td>0.629412</td>\n",
       "      <td>0.571429</td>\n",
       "      <td>0.476407</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_tok</th>\n",
       "      <td>0.723514</td>\n",
       "      <td>0.625000</td>\n",
       "      <td>0.492857</td>\n",
       "      <td>0.461258</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_yap</th>\n",
       "      <td>0.714204</td>\n",
       "      <td>0.625000</td>\n",
       "      <td>0.452381</td>\n",
       "      <td>0.471672</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_tok</th>\n",
       "      <td>0.685318</td>\n",
       "      <td>0.553382</td>\n",
       "      <td>0.361905</td>\n",
       "      <td>0.376896</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_yap</th>\n",
       "      <td>0.705938</td>\n",
       "      <td>0.601161</td>\n",
       "      <td>0.647619</td>\n",
       "      <td>0.393071</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>no_word</th>\n",
       "      <td>0.622339</td>\n",
       "      <td>0.411214</td>\n",
       "      <td>-0.333333</td>\n",
       "      <td>0.084699</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "unk_type                                           ()      (T,)   (T, TM)  \\\n",
       "gold_name        unit  arch      w_embed                                    \n",
       "morph_dev_gold   morph char_cnn  ft_oov_tok  0.754281  0.677696  0.732143   \n",
       "                                 ft_oov_yap  0.753601  0.705882  0.628571   \n",
       "                                 ft_tok      0.757968  0.696315  0.627778   \n",
       "                                 ft_yap      0.759375  0.713072  0.594048   \n",
       "                                 glv_tok     0.737404  0.650245  0.422619   \n",
       "                                 glv_yap     0.751105  0.683893  0.551429   \n",
       "                                 no_word     0.696903  0.595184 -0.389286   \n",
       "                       char_lstm ft_oov_tok  0.757239  0.692565  0.696429   \n",
       "                                 ft_oov_yap  0.754131  0.713072  0.571429   \n",
       "                                 ft_tok      0.757545  0.700178  0.635714   \n",
       "                                 ft_yap      0.754693  0.709150  0.559524   \n",
       "                                 glv_tok     0.733897  0.682190  0.608333   \n",
       "                                 glv_yap     0.746523  0.710997  0.661905   \n",
       "                                 no_word     0.674480  0.525507 -0.433333   \n",
       "                       no_char   ft_oov_tok  0.763919  0.698039  0.700000   \n",
       "                                 ft_oov_yap  0.760657  0.713072  0.607143   \n",
       "                                 ft_tok      0.767361  0.701961  0.621429   \n",
       "                                 ft_yap      0.766936  0.705882  0.535714   \n",
       "                                 glv_tok     0.733222  0.639134  0.528571   \n",
       "                                 glv_yap     0.746239  0.686144  0.700000   \n",
       "                                 no_word     0.688105  0.484172 -0.233333   \n",
       "morph_dev_pruned morph char_cnn  ft_oov_tok  0.733686  0.673775  0.502381   \n",
       "                                 ft_oov_yap  0.735008  0.694118  0.628571   \n",
       "                                 ft_tok      0.739054  0.681527  0.248810   \n",
       "                                 ft_yap      0.726999  0.705229  0.373810   \n",
       "                                 glv_tok     0.724567  0.627292  0.510714   \n",
       "                                 glv_yap     0.735896  0.676539  0.389524   \n",
       "                                 no_word     0.674766  0.572472 -0.380952   \n",
       "                       char_lstm ft_oov_tok  0.740776  0.615833  0.500000   \n",
       "                                 ft_oov_yap  0.742663  0.709150  0.571429   \n",
       "                                 ft_tok      0.721855  0.688368  0.423810   \n",
       "                                 ft_yap      0.741256  0.629412  0.354762   \n",
       "                                 glv_tok     0.720376  0.653655  0.608333   \n",
       "                                 glv_yap     0.730325  0.706422  0.666667   \n",
       "                                 no_word     0.654506  0.525817 -0.433333   \n",
       "                       no_char   ft_oov_tok  0.744478  0.701961  0.471429   \n",
       "                                 ft_oov_yap  0.733918  0.618137  0.328571   \n",
       "                                 ft_tok      0.743230  0.621324  0.048810   \n",
       "                                 ft_yap      0.752647  0.610703  0.376190   \n",
       "                                 glv_tok     0.727446  0.560392  0.340476   \n",
       "                                 glv_yap     0.742650  0.655394  0.566667   \n",
       "                                 no_word     0.661586  0.513313 -0.400000   \n",
       "morph_dev_yap    morph char_cnn  ft_oov_tok  0.716405  0.593824  0.628571   \n",
       "                                 ft_oov_yap  0.711427  0.613971  0.621429   \n",
       "                                 ft_tok      0.709761  0.601618  0.577778   \n",
       "                                 ft_yap      0.711741  0.629412  0.552381   \n",
       "                                 glv_tok     0.685922  0.546046  0.422619   \n",
       "                                 glv_yap     0.702825  0.599232  0.389524   \n",
       "                                 no_word     0.638431  0.519281 -0.285714   \n",
       "                       char_lstm ft_oov_tok  0.716419  0.615833  0.571429   \n",
       "                                 ft_oov_yap  0.713452  0.633088  0.571429   \n",
       "                                 ft_tok      0.703021  0.611471  0.532143   \n",
       "                                 ft_yap      0.701572  0.629412  0.500000   \n",
       "                                 glv_tok     0.681397  0.579869  0.581667   \n",
       "                                 glv_yap     0.701187  0.645899  0.666667   \n",
       "                                 no_word     0.626685  0.436242 -0.433333   \n",
       "                       no_char   ft_oov_tok  0.720240  0.617647  0.564286   \n",
       "                                 ft_oov_yap  0.720024  0.629412  0.571429   \n",
       "                                 ft_tok      0.723514  0.625000  0.492857   \n",
       "                                 ft_yap      0.714204  0.625000  0.452381   \n",
       "                                 glv_tok     0.685318  0.553382  0.361905   \n",
       "                                 glv_yap     0.705938  0.601161  0.647619   \n",
       "                                 no_word     0.622339  0.411214 -0.333333   \n",
       "\n",
       "unk_type                                        (TM,)  \n",
       "gold_name        unit  arch      w_embed               \n",
       "morph_dev_gold   morph char_cnn  ft_oov_tok  0.530702  \n",
       "                                 ft_oov_yap  0.553281  \n",
       "                                 ft_tok      0.530970  \n",
       "                                 ft_yap      0.534189  \n",
       "                                 glv_tok     0.426602  \n",
       "                                 glv_yap     0.450434  \n",
       "                                 no_word     0.226856  \n",
       "                       char_lstm ft_oov_tok  0.544201  \n",
       "                                 ft_oov_yap  0.545209  \n",
       "                                 ft_tok      0.520620  \n",
       "                                 ft_yap      0.562202  \n",
       "                                 glv_tok     0.433548  \n",
       "                                 glv_yap     0.478477  \n",
       "                                 no_word     0.214480  \n",
       "                       no_char   ft_oov_tok  0.509599  \n",
       "                                 ft_oov_yap  0.530753  \n",
       "                                 ft_tok      0.534548  \n",
       "                                 ft_yap      0.545916  \n",
       "                                 glv_tok     0.442740  \n",
       "                                 glv_yap     0.458695  \n",
       "                                 no_word     0.140628  \n",
       "morph_dev_pruned morph char_cnn  ft_oov_tok  0.456523  \n",
       "                                 ft_oov_yap  0.501492  \n",
       "                                 ft_tok      0.497062  \n",
       "                                 ft_yap      0.517877  \n",
       "                                 glv_tok     0.359413  \n",
       "                                 glv_yap     0.376554  \n",
       "                                 no_word     0.171393  \n",
       "                       char_lstm ft_oov_tok  0.463889  \n",
       "                                 ft_oov_yap  0.521660  \n",
       "                                 ft_tok      0.468025  \n",
       "                                 ft_yap      0.459583  \n",
       "                                 glv_tok     0.361926  \n",
       "                                 glv_yap     0.418184  \n",
       "                                 no_word     0.117691  \n",
       "                       no_char   ft_oov_tok  0.440417  \n",
       "                                 ft_oov_yap  0.501661  \n",
       "                                 ft_tok      0.508105  \n",
       "                                 ft_yap      0.545437  \n",
       "                                 glv_tok     0.360642  \n",
       "                                 glv_yap     0.391374  \n",
       "                                 no_word     0.120680  \n",
       "morph_dev_yap    morph char_cnn  ft_oov_tok  0.458156  \n",
       "                                 ft_oov_yap  0.490028  \n",
       "                                 ft_tok      0.462495  \n",
       "                                 ft_yap      0.463550  \n",
       "                                 glv_tok     0.355605  \n",
       "                                 glv_yap     0.374200  \n",
       "                                 no_word     0.192721  \n",
       "                       char_lstm ft_oov_tok  0.447852  \n",
       "                                 ft_oov_yap  0.483582  \n",
       "                                 ft_tok      0.444307  \n",
       "                                 ft_yap      0.478954  \n",
       "                                 glv_tok     0.368878  \n",
       "                                 glv_yap     0.399180  \n",
       "                                 no_word     0.141314  \n",
       "                       no_char   ft_oov_tok  0.421550  \n",
       "                                 ft_oov_yap  0.476407  \n",
       "                                 ft_tok      0.461258  \n",
       "                                 ft_yap      0.471672  \n",
       "                                 glv_tok     0.376896  \n",
       "                                 glv_yap     0.393071  \n",
       "                                 no_word     0.084699  "
      ]
     },
     "execution_count": 144,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gsc = group_scores(scores4, groupers=[ut_gm, ignore, ignore], ment_grouper=ignore, ment_cat_grouper=same)\n",
    "(gsc[(gsc.gold_name.str.contains('morph_dev')) & (gsc.ment_cat=='ORG')]\n",
    " .groupby(['gold_name', 'unit', 'arch', 'w_embed', 'unk_type'])\n",
    " .f.mean().unstack())\n",
    "#0.805956\t0.665775\t0.521717\t0.779888"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Answer is NO - ORG does have more T's but score is actually better in T and T,TM for ORG than for the rest"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## THIS!!!\n",
    "for morph, ft_oov_yap is significantly better on OOEV than ft_oov_tok, while it is slightly worse on IEV. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 151,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>ooev</th>\n",
       "      <th>False</th>\n",
       "      <th>True</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>gold_name</th>\n",
       "      <th>unit</th>\n",
       "      <th>arch</th>\n",
       "      <th>w_embed</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"21\" valign=\"top\">morph_dev_gold</th>\n",
       "      <th rowspan=\"21\" valign=\"top\">morph</th>\n",
       "      <th rowspan=\"7\" valign=\"top\">char_cnn</th>\n",
       "      <th>ft_oov_tok</th>\n",
       "      <td>0.803294</td>\n",
       "      <td>0.752472</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_oov_yap</th>\n",
       "      <td>0.800105</td>\n",
       "      <td>0.801235</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_tok</th>\n",
       "      <td>0.801067</td>\n",
       "      <td>0.738266</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_yap</th>\n",
       "      <td>0.798384</td>\n",
       "      <td>0.724326</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_tok</th>\n",
       "      <td>0.781946</td>\n",
       "      <td>0.758756</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_yap</th>\n",
       "      <td>0.782598</td>\n",
       "      <td>0.763396</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>no_word</th>\n",
       "      <td>NaN</td>\n",
       "      <td>0.630313</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"7\" valign=\"top\">char_lstm</th>\n",
       "      <th>ft_oov_tok</th>\n",
       "      <td>0.799784</td>\n",
       "      <td>0.751506</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_oov_yap</th>\n",
       "      <td>0.797692</td>\n",
       "      <td>0.792757</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_tok</th>\n",
       "      <td>0.795577</td>\n",
       "      <td>0.729034</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_yap</th>\n",
       "      <td>0.796089</td>\n",
       "      <td>0.724924</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_tok</th>\n",
       "      <td>0.780353</td>\n",
       "      <td>0.765531</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_yap</th>\n",
       "      <td>0.787229</td>\n",
       "      <td>0.779572</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>no_word</th>\n",
       "      <td>NaN</td>\n",
       "      <td>0.628893</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"7\" valign=\"top\">no_char</th>\n",
       "      <th>ft_oov_tok</th>\n",
       "      <td>0.804740</td>\n",
       "      <td>0.738246</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_oov_yap</th>\n",
       "      <td>0.800819</td>\n",
       "      <td>0.769181</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_tok</th>\n",
       "      <td>0.798702</td>\n",
       "      <td>0.675565</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_yap</th>\n",
       "      <td>0.798172</td>\n",
       "      <td>0.637938</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_tok</th>\n",
       "      <td>0.774054</td>\n",
       "      <td>0.679241</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_yap</th>\n",
       "      <td>0.777644</td>\n",
       "      <td>0.663680</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>no_word</th>\n",
       "      <td>NaN</td>\n",
       "      <td>0.597824</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"21\" valign=\"top\">morph_dev_pruned</th>\n",
       "      <th rowspan=\"21\" valign=\"top\">morph</th>\n",
       "      <th rowspan=\"7\" valign=\"top\">char_cnn</th>\n",
       "      <th>ft_oov_tok</th>\n",
       "      <td>0.781063</td>\n",
       "      <td>0.734953</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_oov_yap</th>\n",
       "      <td>0.781268</td>\n",
       "      <td>0.781371</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_tok</th>\n",
       "      <td>0.769256</td>\n",
       "      <td>0.710554</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_yap</th>\n",
       "      <td>0.780773</td>\n",
       "      <td>0.686235</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_tok</th>\n",
       "      <td>0.755051</td>\n",
       "      <td>0.727229</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_yap</th>\n",
       "      <td>0.762051</td>\n",
       "      <td>0.745924</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>no_word</th>\n",
       "      <td>NaN</td>\n",
       "      <td>0.591396</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"7\" valign=\"top\">char_lstm</th>\n",
       "      <th>ft_oov_tok</th>\n",
       "      <td>0.775301</td>\n",
       "      <td>0.720980</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_oov_yap</th>\n",
       "      <td>0.781342</td>\n",
       "      <td>0.777014</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_tok</th>\n",
       "      <td>0.761941</td>\n",
       "      <td>0.704041</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_yap</th>\n",
       "      <td>0.775658</td>\n",
       "      <td>0.701529</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_tok</th>\n",
       "      <td>0.754236</td>\n",
       "      <td>0.706132</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_yap</th>\n",
       "      <td>0.768953</td>\n",
       "      <td>0.727962</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>no_word</th>\n",
       "      <td>NaN</td>\n",
       "      <td>0.591850</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"7\" valign=\"top\">no_char</th>\n",
       "      <th>ft_oov_tok</th>\n",
       "      <td>0.783031</td>\n",
       "      <td>0.714378</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_oov_yap</th>\n",
       "      <td>0.779654</td>\n",
       "      <td>0.703037</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_tok</th>\n",
       "      <td>0.765577</td>\n",
       "      <td>0.660737</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_yap</th>\n",
       "      <td>0.785093</td>\n",
       "      <td>0.595679</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_tok</th>\n",
       "      <td>0.749318</td>\n",
       "      <td>0.652870</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_yap</th>\n",
       "      <td>0.760127</td>\n",
       "      <td>0.590709</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>no_word</th>\n",
       "      <td>NaN</td>\n",
       "      <td>0.553703</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"21\" valign=\"top\">morph_dev_yap</th>\n",
       "      <th rowspan=\"21\" valign=\"top\">morph</th>\n",
       "      <th rowspan=\"7\" valign=\"top\">char_cnn</th>\n",
       "      <th>ft_oov_tok</th>\n",
       "      <td>0.734973</td>\n",
       "      <td>0.640723</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_oov_yap</th>\n",
       "      <td>0.728185</td>\n",
       "      <td>0.710992</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_tok</th>\n",
       "      <td>0.728827</td>\n",
       "      <td>0.619247</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_yap</th>\n",
       "      <td>0.721258</td>\n",
       "      <td>0.633719</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_tok</th>\n",
       "      <td>0.711112</td>\n",
       "      <td>0.627565</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_yap</th>\n",
       "      <td>0.708551</td>\n",
       "      <td>0.636644</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>no_word</th>\n",
       "      <td>NaN</td>\n",
       "      <td>0.567970</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"7\" valign=\"top\">char_lstm</th>\n",
       "      <th>ft_oov_tok</th>\n",
       "      <td>0.728251</td>\n",
       "      <td>0.636989</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_oov_yap</th>\n",
       "      <td>0.725335</td>\n",
       "      <td>0.704232</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_tok</th>\n",
       "      <td>0.720579</td>\n",
       "      <td>0.614307</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_yap</th>\n",
       "      <td>0.716322</td>\n",
       "      <td>0.624931</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_tok</th>\n",
       "      <td>0.708154</td>\n",
       "      <td>0.632977</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_yap</th>\n",
       "      <td>0.714142</td>\n",
       "      <td>0.636351</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>no_word</th>\n",
       "      <td>NaN</td>\n",
       "      <td>0.570697</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"7\" valign=\"top\">no_char</th>\n",
       "      <th>ft_oov_tok</th>\n",
       "      <td>0.732080</td>\n",
       "      <td>0.627821</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_oov_yap</th>\n",
       "      <td>0.730928</td>\n",
       "      <td>0.683529</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_tok</th>\n",
       "      <td>0.726145</td>\n",
       "      <td>0.569184</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_yap</th>\n",
       "      <td>0.725599</td>\n",
       "      <td>0.537925</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_tok</th>\n",
       "      <td>0.705431</td>\n",
       "      <td>0.563898</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_yap</th>\n",
       "      <td>0.706231</td>\n",
       "      <td>0.543455</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>no_word</th>\n",
       "      <td>NaN</td>\n",
       "      <td>0.539510</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "ooev                                            False     True \n",
       "gold_name        unit  arch      w_embed                       \n",
       "morph_dev_gold   morph char_cnn  ft_oov_tok  0.803294  0.752472\n",
       "                                 ft_oov_yap  0.800105  0.801235\n",
       "                                 ft_tok      0.801067  0.738266\n",
       "                                 ft_yap      0.798384  0.724326\n",
       "                                 glv_tok     0.781946  0.758756\n",
       "                                 glv_yap     0.782598  0.763396\n",
       "                                 no_word          NaN  0.630313\n",
       "                       char_lstm ft_oov_tok  0.799784  0.751506\n",
       "                                 ft_oov_yap  0.797692  0.792757\n",
       "                                 ft_tok      0.795577  0.729034\n",
       "                                 ft_yap      0.796089  0.724924\n",
       "                                 glv_tok     0.780353  0.765531\n",
       "                                 glv_yap     0.787229  0.779572\n",
       "                                 no_word          NaN  0.628893\n",
       "                       no_char   ft_oov_tok  0.804740  0.738246\n",
       "                                 ft_oov_yap  0.800819  0.769181\n",
       "                                 ft_tok      0.798702  0.675565\n",
       "                                 ft_yap      0.798172  0.637938\n",
       "                                 glv_tok     0.774054  0.679241\n",
       "                                 glv_yap     0.777644  0.663680\n",
       "                                 no_word          NaN  0.597824\n",
       "morph_dev_pruned morph char_cnn  ft_oov_tok  0.781063  0.734953\n",
       "                                 ft_oov_yap  0.781268  0.781371\n",
       "                                 ft_tok      0.769256  0.710554\n",
       "                                 ft_yap      0.780773  0.686235\n",
       "                                 glv_tok     0.755051  0.727229\n",
       "                                 glv_yap     0.762051  0.745924\n",
       "                                 no_word          NaN  0.591396\n",
       "                       char_lstm ft_oov_tok  0.775301  0.720980\n",
       "                                 ft_oov_yap  0.781342  0.777014\n",
       "                                 ft_tok      0.761941  0.704041\n",
       "                                 ft_yap      0.775658  0.701529\n",
       "                                 glv_tok     0.754236  0.706132\n",
       "                                 glv_yap     0.768953  0.727962\n",
       "                                 no_word          NaN  0.591850\n",
       "                       no_char   ft_oov_tok  0.783031  0.714378\n",
       "                                 ft_oov_yap  0.779654  0.703037\n",
       "                                 ft_tok      0.765577  0.660737\n",
       "                                 ft_yap      0.785093  0.595679\n",
       "                                 glv_tok     0.749318  0.652870\n",
       "                                 glv_yap     0.760127  0.590709\n",
       "                                 no_word          NaN  0.553703\n",
       "morph_dev_yap    morph char_cnn  ft_oov_tok  0.734973  0.640723\n",
       "                                 ft_oov_yap  0.728185  0.710992\n",
       "                                 ft_tok      0.728827  0.619247\n",
       "                                 ft_yap      0.721258  0.633719\n",
       "                                 glv_tok     0.711112  0.627565\n",
       "                                 glv_yap     0.708551  0.636644\n",
       "                                 no_word          NaN  0.567970\n",
       "                       char_lstm ft_oov_tok  0.728251  0.636989\n",
       "                                 ft_oov_yap  0.725335  0.704232\n",
       "                                 ft_tok      0.720579  0.614307\n",
       "                                 ft_yap      0.716322  0.624931\n",
       "                                 glv_tok     0.708154  0.632977\n",
       "                                 glv_yap     0.714142  0.636351\n",
       "                                 no_word          NaN  0.570697\n",
       "                       no_char   ft_oov_tok  0.732080  0.627821\n",
       "                                 ft_oov_yap  0.730928  0.683529\n",
       "                                 ft_tok      0.726145  0.569184\n",
       "                                 ft_yap      0.725599  0.537925\n",
       "                                 glv_tok     0.705431  0.563898\n",
       "                                 glv_yap     0.706231  0.543455\n",
       "                                 no_word          NaN  0.539510"
      ]
     },
     "execution_count": 151,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gsc = group_scores(scores5, groupers=[ignore, ignore, max, ignore], ment_grouper=ignore)\n",
    "gsc[gsc.gold_name.str.contains('morph_dev')].fillna('x').groupby(['gold_name', 'unit', 'arch', 'w_embed', 'ooev'], sort=False).f.mean().unstack().sort_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 295,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_cm(s):\n",
    "    if s.unit=='morph':\n",
    "        if '_yap' in s.w_embed:\n",
    "            return 'Match'\n",
    "        else:\n",
    "            return 'Clash'\n",
    "    else:\n",
    "        if '_tok' in s.w_embed:\n",
    "            return 'Match'\n",
    "        else:\n",
    "            return 'Clash'\n",
    "    return 'na'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 297,
   "metadata": {},
   "outputs": [],
   "source": [
    "gsc = (group_scores(scores5, groupers=[ignore, ignore, max, ignore], ment_grouper=ignore, ment_cat_grouper=ignore))\n",
    "gsc['cm'] = gsc.apply(get_cm, axis=1)\n",
    "gsc['embed_type'] = gsc.w_embed.str.replace('_tok|_yap', '')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 302,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr:last-of-type th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>embed_type</th>\n",
       "      <th colspan=\"2\" halign=\"left\">glv</th>\n",
       "      <th colspan=\"2\" halign=\"left\">ft</th>\n",
       "      <th colspan=\"2\" halign=\"left\">ft_oov</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>ooev</th>\n",
       "      <th>False</th>\n",
       "      <th>True</th>\n",
       "      <th>False</th>\n",
       "      <th>True</th>\n",
       "      <th>False</th>\n",
       "      <th>True</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>gold_name</th>\n",
       "      <th>unit</th>\n",
       "      <th>cm</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">morph_dev_gold</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">morph</th>\n",
       "      <th>Clash</th>\n",
       "      <td>$78.19 ± 0.6$</td>\n",
       "      <td>$75.88 ± 2.3$</td>\n",
       "      <td>$80.11 ± 0.8$</td>\n",
       "      <td>$73.83 ± 3.8$</td>\n",
       "      <td>$80.33 ± 0.6$</td>\n",
       "      <td>$75.25 ± 1.8$</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Match</th>\n",
       "      <td>$78.23 ± 0.6$</td>\n",
       "      <td>$76.34 ± 3.3$</td>\n",
       "      <td>$79.84 ± 0.4$</td>\n",
       "      <td>$72.43 ± 2.2$</td>\n",
       "      <td>$80.01 ± 0.6$</td>\n",
       "      <td>$80.12 ± 2.3$</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">morph_dev_pruned</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">morph</th>\n",
       "      <th>Clash</th>\n",
       "      <td>$75.51 ± 0.9$</td>\n",
       "      <td>$72.72 ± 2.6$</td>\n",
       "      <td>$76.93 ± 1.0$</td>\n",
       "      <td>$71.06 ± 3.7$</td>\n",
       "      <td>$78.11 ± 0.7$</td>\n",
       "      <td>$73.50 ± 2.3$</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Match</th>\n",
       "      <td>$76.21 ± 0.5$</td>\n",
       "      <td>$74.59 ± 3.0$</td>\n",
       "      <td>$78.08 ± 0.3$</td>\n",
       "      <td>$68.62 ± 1.9$</td>\n",
       "      <td>$78.13 ± 0.6$</td>\n",
       "      <td>$78.14 ± 2.2$</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"4\" valign=\"top\">token_dev</th>\n",
       "      <th rowspan=\"2\" valign=\"top\">multitok</th>\n",
       "      <th>Clash</th>\n",
       "      <td>$74.62 ± 0.9$</td>\n",
       "      <td>$56.17 ± 2.1$</td>\n",
       "      <td>$78.13 ± 1.3$</td>\n",
       "      <td>$56.18 ± 1.1$</td>\n",
       "      <td>$78.16 ± 1.2$</td>\n",
       "      <td>$69.71 ± 1.4$</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Match</th>\n",
       "      <td>$74.26 ± 0.8$</td>\n",
       "      <td>$62.79 ± 2.4$</td>\n",
       "      <td>$77.02 ± 0.8$</td>\n",
       "      <td>$62.90 ± 3.1$</td>\n",
       "      <td>$77.52 ± 0.9$</td>\n",
       "      <td>$77.58 ± 2.6$</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">token</th>\n",
       "      <th>Clash</th>\n",
       "      <td>$75.37 ± 1.1$</td>\n",
       "      <td>$55.97 ± 2.0$</td>\n",
       "      <td>$77.67 ± 1.1$</td>\n",
       "      <td>$54.67 ± 1.7$</td>\n",
       "      <td>$78.92 ± 0.7$</td>\n",
       "      <td>$69.26 ± 1.3$</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Match</th>\n",
       "      <td>$74.95 ± 0.7$</td>\n",
       "      <td>$68.80 ± 2.4$</td>\n",
       "      <td>$78.16 ± 0.6$</td>\n",
       "      <td>$67.76 ± 4.2$</td>\n",
       "      <td>$78.44 ± 0.6$</td>\n",
       "      <td>$76.36 ± 2.7$</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "embed_type                                 glv                            ft  \\\n",
       "ooev                                     False          True           False   \n",
       "gold_name        unit     cm                                                   \n",
       "morph_dev_gold   morph    Clash  $78.19 ± 0.6$  $75.88 ± 2.3$  $80.11 ± 0.8$   \n",
       "                          Match  $78.23 ± 0.6$  $76.34 ± 3.3$  $79.84 ± 0.4$   \n",
       "morph_dev_pruned morph    Clash  $75.51 ± 0.9$  $72.72 ± 2.6$  $76.93 ± 1.0$   \n",
       "                          Match  $76.21 ± 0.5$  $74.59 ± 3.0$  $78.08 ± 0.3$   \n",
       "token_dev        multitok Clash  $74.62 ± 0.9$  $56.17 ± 2.1$  $78.13 ± 1.3$   \n",
       "                          Match  $74.26 ± 0.8$  $62.79 ± 2.4$  $77.02 ± 0.8$   \n",
       "                 token    Clash  $75.37 ± 1.1$  $55.97 ± 2.0$  $77.67 ± 1.1$   \n",
       "                          Match  $74.95 ± 0.7$  $68.80 ± 2.4$  $78.16 ± 0.6$   \n",
       "\n",
       "embed_type                                             ft_oov                 \n",
       "ooev                                     True           False          True   \n",
       "gold_name        unit     cm                                                  \n",
       "morph_dev_gold   morph    Clash  $73.83 ± 3.8$  $80.33 ± 0.6$  $75.25 ± 1.8$  \n",
       "                          Match  $72.43 ± 2.2$  $80.01 ± 0.6$  $80.12 ± 2.3$  \n",
       "morph_dev_pruned morph    Clash  $71.06 ± 3.7$  $78.11 ± 0.7$  $73.50 ± 2.3$  \n",
       "                          Match  $68.62 ± 1.9$  $78.13 ± 0.6$  $78.14 ± 2.2$  \n",
       "token_dev        multitok Clash  $56.18 ± 1.1$  $78.16 ± 1.2$  $69.71 ± 1.4$  \n",
       "                          Match  $62.90 ± 3.1$  $77.52 ± 0.9$  $77.58 ± 2.6$  \n",
       "                 token    Clash  $54.67 ± 1.7$  $78.92 ± 0.7$  $69.26 ± 1.3$  \n",
       "                          Match  $67.76 ± 4.2$  $78.44 ± 0.6$  $76.36 ± 2.7$  "
      ]
     },
     "execution_count": 302,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x = (gsc[((gsc.gold_name.isin(['morph_dev_gold','morph_dev_pruned']))\n",
    "      & (gsc.arch=='char_cnn')\n",
    "      & (gsc.w_embed!='no_word')\n",
    "\n",
    "     )\n",
    "    |\n",
    "     (\n",
    "        (gsc.gold_name.isin(['token_dev']))\n",
    "      & (gsc.arch=='char_lstm')\n",
    "      & (gsc.w_embed!='no_word')\n",
    "\n",
    "     )]\n",
    " .groupby(['gold_name', 'unit', 'cm', 'embed_type', 'ooev'])\n",
    " .f.agg(['mean', 'std']).mul(100).round(2)\n",
    "         .assign(mean = lambda x: '$'+x['mean'].apply('{:,.2f}'.format).astype(str)+' ± '+ x['std'].round(1).astype(str)+'$')[['mean']]\n",
    "         .unstack([-2,-1]))\n",
    "x.columns = x.columns.droplevel(0)\n",
    "x[[('glv', False), ('glv', True), ('ft', False), \n",
    "       ('ft', True), ('ft_oov', False), ('ft_oov', True)]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 303,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\\begin{tabular}{lllllllll}\n",
      "\\toprule\n",
      "          &       & \\textbf{embed\\_type} & \\multicolumn{2}{l}{glv} & \\multicolumn{2}{l}{ft} & \\multicolumn{2}{l}{ft\\_oov} \\\\\n",
      "          &       & \\textbf{ooev} &          False &          True  &          False &          True  &          False &          True  \\\\\n",
      "\\textbf{gold\\_name} & \\textbf{unit} & \\textbf{cm} &                &                &                &                &                &                \\\\\n",
      "\\midrule\n",
      "\\textbf{morph\\_dev\\_gold} & \\textbf{morph} & \\textbf{Clash} &  $78.19 \\pm 0.6$ &  $75.88 \\pm 2.3$ &  $80.11 \\pm 0.8$ &  $73.83 \\pm 3.8$ &  $80.33 \\pm 0.6$ &  $75.25 \\pm 1.8$ \\\\\n",
      "          &       & \\textbf{Match} &  $78.23 \\pm 0.6$ &  $76.34 \\pm 3.3$ &  $79.84 \\pm 0.4$ &  $72.43 \\pm 2.2$ &  $80.01 \\pm 0.6$ &  $80.12 \\pm 2.3$ \\\\\n",
      "\\textbf{morph\\_dev\\_pruned} & \\textbf{morph} & \\textbf{Clash} &  $75.51 \\pm 0.9$ &  $72.72 \\pm 2.6$ &  $76.93 \\pm 1.0$ &  $71.06 \\pm 3.7$ &  $78.11 \\pm 0.7$ &  $73.50 \\pm 2.3$ \\\\\n",
      "          &       & \\textbf{Match} &  $76.21 \\pm 0.5$ &  $74.59 \\pm 3.0$ &  $78.08 \\pm 0.3$ &  $68.62 \\pm 1.9$ &  $78.13 \\pm 0.6$ &  $78.14 \\pm 2.2$ \\\\\n",
      "\\textbf{token\\_dev} & \\textbf{multitok} & \\textbf{Clash} &  $74.62 \\pm 0.9$ &  $56.17 \\pm 2.1$ &  $78.13 \\pm 1.3$ &  $56.18 \\pm 1.1$ &  $78.16 \\pm 1.2$ &  $69.71 \\pm 1.4$ \\\\\n",
      "          &       & \\textbf{Match} &  $74.26 \\pm 0.8$ &  $62.79 \\pm 2.4$ &  $77.02 \\pm 0.8$ &  $62.90 \\pm 3.1$ &  $77.52 \\pm 0.9$ &  $77.58 \\pm 2.6$ \\\\\n",
      "          & \\textbf{token} & \\textbf{Clash} &  $75.37 \\pm 1.1$ &  $55.97 \\pm 2.0$ &  $77.67 \\pm 1.1$ &  $54.67 \\pm 1.7$ &  $78.92 \\pm 0.7$ &  $69.26 \\pm 1.3$ \\\\\n",
      "          &       & \\textbf{Match} &  $74.95 \\pm 0.7$ &  $68.80 \\pm 2.4$ &  $78.16 \\pm 0.6$ &  $67.76 \\pm 4.2$ &  $78.44 \\pm 0.6$ &  $76.36 \\pm 2.7$ \\\\\n",
      "\\bottomrule\n",
      "\\end{tabular}\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print (x[[('glv', False), ('glv', True), ('ft', False), \n",
    "       ('ft', True), ('ft_oov', False), ('ft_oov', True)]]\n",
    "       .to_latex(bold_rows=True).replace('±', '\\pm').replace('\\$', '$'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 275,
   "metadata": {},
   "outputs": [],
   "source": [
    "from scipy.stats import ttest_ind"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 283,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Ttest_indResult(statistic=2.5674395676052857, pvalue=0.01937938613779738)"
      ]
     },
     "execution_count": 283,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tt = (gsc[(gsc.unk_type=='Comp') & ( ((gsc.gold_name=='morph_dev_pruned')\n",
    "      & (gsc.arch=='char_cnn')\n",
    "      & (gsc.w_embed=='ft_oov_yap')\n",
    "\n",
    "     )\n",
    "    |\n",
    "     (\n",
    "        (gsc.gold_name.isin(['token_dev']))\n",
    "      & (gsc.arch=='char_lstm')\n",
    "      & (gsc.w_embed=='ft_oov_tok')\n",
    "\n",
    "     ))]\n",
    ")\n",
    "\n",
    "ttest_ind(tt[tt.unit=='morph'].f, tt[tt.unit=='token'].f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 152,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr:last-of-type th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th colspan=\"2\" halign=\"left\">true</th>\n",
       "      <th colspan=\"2\" halign=\"left\">pred</th>\n",
       "      <th colspan=\"2\" halign=\"left\">correct</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>ooev</th>\n",
       "      <th>False</th>\n",
       "      <th>True</th>\n",
       "      <th>False</th>\n",
       "      <th>True</th>\n",
       "      <th>False</th>\n",
       "      <th>True</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>gold_name</th>\n",
       "      <th>unit</th>\n",
       "      <th>arch</th>\n",
       "      <th>w_embed</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"21\" valign=\"top\">morph_dev_gold</th>\n",
       "      <th rowspan=\"21\" valign=\"top\">morph</th>\n",
       "      <th rowspan=\"7\" valign=\"top\">char_cnn</th>\n",
       "      <th>ft_oov_tok</th>\n",
       "      <td>441.0</td>\n",
       "      <td>58.0</td>\n",
       "      <td>399.8</td>\n",
       "      <td>56.3</td>\n",
       "      <td>337.7</td>\n",
       "      <td>43.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_oov_yap</th>\n",
       "      <td>434.0</td>\n",
       "      <td>65.0</td>\n",
       "      <td>385.9</td>\n",
       "      <td>63.3</td>\n",
       "      <td>328.0</td>\n",
       "      <td>51.4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_tok</th>\n",
       "      <td>441.0</td>\n",
       "      <td>58.0</td>\n",
       "      <td>395.9</td>\n",
       "      <td>50.3</td>\n",
       "      <td>335.2</td>\n",
       "      <td>40.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_yap</th>\n",
       "      <td>434.0</td>\n",
       "      <td>65.0</td>\n",
       "      <td>379.4</td>\n",
       "      <td>50.1</td>\n",
       "      <td>324.7</td>\n",
       "      <td>41.7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_tok</th>\n",
       "      <td>441.0</td>\n",
       "      <td>58.0</td>\n",
       "      <td>391.8</td>\n",
       "      <td>50.0</td>\n",
       "      <td>325.6</td>\n",
       "      <td>41.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_yap</th>\n",
       "      <td>434.0</td>\n",
       "      <td>65.0</td>\n",
       "      <td>386.6</td>\n",
       "      <td>54.4</td>\n",
       "      <td>321.1</td>\n",
       "      <td>45.6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>no_word</th>\n",
       "      <td>NaN</td>\n",
       "      <td>499.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>331.4</td>\n",
       "      <td>NaN</td>\n",
       "      <td>261.7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"7\" valign=\"top\">char_lstm</th>\n",
       "      <th>ft_oov_tok</th>\n",
       "      <td>441.0</td>\n",
       "      <td>58.0</td>\n",
       "      <td>404.5</td>\n",
       "      <td>55.9</td>\n",
       "      <td>338.1</td>\n",
       "      <td>42.8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_oov_yap</th>\n",
       "      <td>434.0</td>\n",
       "      <td>65.0</td>\n",
       "      <td>383.6</td>\n",
       "      <td>62.4</td>\n",
       "      <td>326.1</td>\n",
       "      <td>50.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_tok</th>\n",
       "      <td>441.0</td>\n",
       "      <td>58.0</td>\n",
       "      <td>400.2</td>\n",
       "      <td>48.4</td>\n",
       "      <td>334.6</td>\n",
       "      <td>38.8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_yap</th>\n",
       "      <td>434.0</td>\n",
       "      <td>65.0</td>\n",
       "      <td>386.5</td>\n",
       "      <td>51.4</td>\n",
       "      <td>326.6</td>\n",
       "      <td>42.2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_tok</th>\n",
       "      <td>441.0</td>\n",
       "      <td>58.0</td>\n",
       "      <td>402.2</td>\n",
       "      <td>51.2</td>\n",
       "      <td>329.0</td>\n",
       "      <td>41.8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_yap</th>\n",
       "      <td>434.0</td>\n",
       "      <td>65.0</td>\n",
       "      <td>400.6</td>\n",
       "      <td>56.1</td>\n",
       "      <td>328.5</td>\n",
       "      <td>47.2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>no_word</th>\n",
       "      <td>NaN</td>\n",
       "      <td>499.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>336.5</td>\n",
       "      <td>NaN</td>\n",
       "      <td>262.7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"7\" valign=\"top\">no_char</th>\n",
       "      <th>ft_oov_tok</th>\n",
       "      <td>441.0</td>\n",
       "      <td>58.0</td>\n",
       "      <td>398.3</td>\n",
       "      <td>54.7</td>\n",
       "      <td>337.7</td>\n",
       "      <td>41.6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_oov_yap</th>\n",
       "      <td>434.0</td>\n",
       "      <td>65.0</td>\n",
       "      <td>384.9</td>\n",
       "      <td>59.8</td>\n",
       "      <td>327.9</td>\n",
       "      <td>48.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_tok</th>\n",
       "      <td>441.0</td>\n",
       "      <td>58.0</td>\n",
       "      <td>400.1</td>\n",
       "      <td>43.0</td>\n",
       "      <td>335.9</td>\n",
       "      <td>34.1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_yap</th>\n",
       "      <td>434.0</td>\n",
       "      <td>65.0</td>\n",
       "      <td>382.1</td>\n",
       "      <td>41.9</td>\n",
       "      <td>325.7</td>\n",
       "      <td>34.1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_tok</th>\n",
       "      <td>441.0</td>\n",
       "      <td>58.0</td>\n",
       "      <td>399.0</td>\n",
       "      <td>41.8</td>\n",
       "      <td>325.1</td>\n",
       "      <td>33.9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_yap</th>\n",
       "      <td>434.0</td>\n",
       "      <td>65.0</td>\n",
       "      <td>392.6</td>\n",
       "      <td>44.6</td>\n",
       "      <td>321.4</td>\n",
       "      <td>36.4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>no_word</th>\n",
       "      <td>NaN</td>\n",
       "      <td>499.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>308.9</td>\n",
       "      <td>NaN</td>\n",
       "      <td>241.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"21\" valign=\"top\">morph_dev_pruned</th>\n",
       "      <th rowspan=\"21\" valign=\"top\">morph</th>\n",
       "      <th rowspan=\"7\" valign=\"top\">char_cnn</th>\n",
       "      <th>ft_oov_tok</th>\n",
       "      <td>440.0</td>\n",
       "      <td>59.0</td>\n",
       "      <td>395.3</td>\n",
       "      <td>55.3</td>\n",
       "      <td>326.2</td>\n",
       "      <td>42.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_oov_yap</th>\n",
       "      <td>434.0</td>\n",
       "      <td>65.0</td>\n",
       "      <td>381.6</td>\n",
       "      <td>61.7</td>\n",
       "      <td>318.6</td>\n",
       "      <td>49.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_tok</th>\n",
       "      <td>438.0</td>\n",
       "      <td>61.0</td>\n",
       "      <td>388.8</td>\n",
       "      <td>50.4</td>\n",
       "      <td>318.0</td>\n",
       "      <td>39.6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_yap</th>\n",
       "      <td>432.0</td>\n",
       "      <td>67.0</td>\n",
       "      <td>373.1</td>\n",
       "      <td>50.1</td>\n",
       "      <td>314.3</td>\n",
       "      <td>40.2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_tok</th>\n",
       "      <td>439.0</td>\n",
       "      <td>60.0</td>\n",
       "      <td>387.7</td>\n",
       "      <td>49.4</td>\n",
       "      <td>312.1</td>\n",
       "      <td>39.8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_yap</th>\n",
       "      <td>436.0</td>\n",
       "      <td>63.0</td>\n",
       "      <td>386.0</td>\n",
       "      <td>52.8</td>\n",
       "      <td>313.2</td>\n",
       "      <td>43.2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>no_word</th>\n",
       "      <td>NaN</td>\n",
       "      <td>499.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>320.1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>242.2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"7\" valign=\"top\">char_lstm</th>\n",
       "      <th>ft_oov_tok</th>\n",
       "      <td>440.0</td>\n",
       "      <td>59.0</td>\n",
       "      <td>398.4</td>\n",
       "      <td>53.9</td>\n",
       "      <td>325.0</td>\n",
       "      <td>40.7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_oov_yap</th>\n",
       "      <td>435.0</td>\n",
       "      <td>64.0</td>\n",
       "      <td>380.0</td>\n",
       "      <td>61.1</td>\n",
       "      <td>318.4</td>\n",
       "      <td>48.6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_tok</th>\n",
       "      <td>441.0</td>\n",
       "      <td>58.0</td>\n",
       "      <td>398.0</td>\n",
       "      <td>47.1</td>\n",
       "      <td>319.6</td>\n",
       "      <td>37.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_yap</th>\n",
       "      <td>435.0</td>\n",
       "      <td>64.0</td>\n",
       "      <td>382.1</td>\n",
       "      <td>50.0</td>\n",
       "      <td>316.9</td>\n",
       "      <td>40.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_tok</th>\n",
       "      <td>440.0</td>\n",
       "      <td>59.0</td>\n",
       "      <td>396.1</td>\n",
       "      <td>50.3</td>\n",
       "      <td>315.3</td>\n",
       "      <td>38.6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_yap</th>\n",
       "      <td>434.0</td>\n",
       "      <td>65.0</td>\n",
       "      <td>398.6</td>\n",
       "      <td>56.2</td>\n",
       "      <td>320.1</td>\n",
       "      <td>44.1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>no_word</th>\n",
       "      <td>NaN</td>\n",
       "      <td>499.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>330.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>245.3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"7\" valign=\"top\">no_char</th>\n",
       "      <th>ft_oov_tok</th>\n",
       "      <td>438.0</td>\n",
       "      <td>61.0</td>\n",
       "      <td>392.9</td>\n",
       "      <td>53.5</td>\n",
       "      <td>325.3</td>\n",
       "      <td>40.9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_oov_yap</th>\n",
       "      <td>430.0</td>\n",
       "      <td>69.0</td>\n",
       "      <td>376.0</td>\n",
       "      <td>59.3</td>\n",
       "      <td>314.2</td>\n",
       "      <td>45.1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_tok</th>\n",
       "      <td>441.0</td>\n",
       "      <td>58.0</td>\n",
       "      <td>391.3</td>\n",
       "      <td>41.9</td>\n",
       "      <td>318.6</td>\n",
       "      <td>33.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_yap</th>\n",
       "      <td>431.0</td>\n",
       "      <td>68.0</td>\n",
       "      <td>374.0</td>\n",
       "      <td>41.4</td>\n",
       "      <td>316.0</td>\n",
       "      <td>32.6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_tok</th>\n",
       "      <td>440.0</td>\n",
       "      <td>59.0</td>\n",
       "      <td>391.7</td>\n",
       "      <td>39.9</td>\n",
       "      <td>311.6</td>\n",
       "      <td>32.3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_yap</th>\n",
       "      <td>429.0</td>\n",
       "      <td>70.0</td>\n",
       "      <td>384.3</td>\n",
       "      <td>47.2</td>\n",
       "      <td>309.1</td>\n",
       "      <td>34.6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>no_word</th>\n",
       "      <td>NaN</td>\n",
       "      <td>499.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>293.1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>219.3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"21\" valign=\"top\">morph_dev_yap</th>\n",
       "      <th rowspan=\"21\" valign=\"top\">morph</th>\n",
       "      <th rowspan=\"7\" valign=\"top\">char_cnn</th>\n",
       "      <th>ft_oov_tok</th>\n",
       "      <td>437.0</td>\n",
       "      <td>62.0</td>\n",
       "      <td>387.0</td>\n",
       "      <td>53.2</td>\n",
       "      <td>302.8</td>\n",
       "      <td>36.9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_oov_yap</th>\n",
       "      <td>435.0</td>\n",
       "      <td>64.0</td>\n",
       "      <td>376.6</td>\n",
       "      <td>58.1</td>\n",
       "      <td>295.5</td>\n",
       "      <td>43.4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_tok</th>\n",
       "      <td>437.0</td>\n",
       "      <td>62.0</td>\n",
       "      <td>386.0</td>\n",
       "      <td>47.8</td>\n",
       "      <td>299.9</td>\n",
       "      <td>34.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_yap</th>\n",
       "      <td>435.0</td>\n",
       "      <td>64.0</td>\n",
       "      <td>373.6</td>\n",
       "      <td>44.2</td>\n",
       "      <td>291.6</td>\n",
       "      <td>34.3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_tok</th>\n",
       "      <td>437.0</td>\n",
       "      <td>62.0</td>\n",
       "      <td>385.1</td>\n",
       "      <td>46.9</td>\n",
       "      <td>292.3</td>\n",
       "      <td>34.2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_yap</th>\n",
       "      <td>435.0</td>\n",
       "      <td>64.0</td>\n",
       "      <td>381.3</td>\n",
       "      <td>51.0</td>\n",
       "      <td>289.2</td>\n",
       "      <td>36.6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>no_word</th>\n",
       "      <td>NaN</td>\n",
       "      <td>499.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>320.4</td>\n",
       "      <td>NaN</td>\n",
       "      <td>232.7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"7\" valign=\"top\">char_lstm</th>\n",
       "      <th>ft_oov_tok</th>\n",
       "      <td>437.0</td>\n",
       "      <td>62.0</td>\n",
       "      <td>392.4</td>\n",
       "      <td>52.6</td>\n",
       "      <td>302.0</td>\n",
       "      <td>36.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_oov_yap</th>\n",
       "      <td>435.0</td>\n",
       "      <td>64.0</td>\n",
       "      <td>374.0</td>\n",
       "      <td>56.7</td>\n",
       "      <td>293.4</td>\n",
       "      <td>42.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_tok</th>\n",
       "      <td>437.0</td>\n",
       "      <td>62.0</td>\n",
       "      <td>392.7</td>\n",
       "      <td>45.4</td>\n",
       "      <td>298.9</td>\n",
       "      <td>33.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_yap</th>\n",
       "      <td>435.0</td>\n",
       "      <td>64.0</td>\n",
       "      <td>381.4</td>\n",
       "      <td>45.8</td>\n",
       "      <td>292.4</td>\n",
       "      <td>34.3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_tok</th>\n",
       "      <td>437.0</td>\n",
       "      <td>62.0</td>\n",
       "      <td>394.5</td>\n",
       "      <td>48.6</td>\n",
       "      <td>294.4</td>\n",
       "      <td>35.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_yap</th>\n",
       "      <td>435.0</td>\n",
       "      <td>64.0</td>\n",
       "      <td>395.7</td>\n",
       "      <td>52.0</td>\n",
       "      <td>296.6</td>\n",
       "      <td>36.9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>no_word</th>\n",
       "      <td>NaN</td>\n",
       "      <td>499.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>325.4</td>\n",
       "      <td>NaN</td>\n",
       "      <td>235.2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"7\" valign=\"top\">no_char</th>\n",
       "      <th>ft_oov_tok</th>\n",
       "      <td>437.0</td>\n",
       "      <td>62.0</td>\n",
       "      <td>385.9</td>\n",
       "      <td>51.4</td>\n",
       "      <td>301.2</td>\n",
       "      <td>35.6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_oov_yap</th>\n",
       "      <td>435.0</td>\n",
       "      <td>64.0</td>\n",
       "      <td>374.9</td>\n",
       "      <td>54.5</td>\n",
       "      <td>296.0</td>\n",
       "      <td>40.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_tok</th>\n",
       "      <td>437.0</td>\n",
       "      <td>62.0</td>\n",
       "      <td>389.3</td>\n",
       "      <td>40.2</td>\n",
       "      <td>300.0</td>\n",
       "      <td>29.1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_yap</th>\n",
       "      <td>435.0</td>\n",
       "      <td>64.0</td>\n",
       "      <td>373.7</td>\n",
       "      <td>37.1</td>\n",
       "      <td>293.4</td>\n",
       "      <td>27.2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_tok</th>\n",
       "      <td>437.0</td>\n",
       "      <td>62.0</td>\n",
       "      <td>390.3</td>\n",
       "      <td>37.6</td>\n",
       "      <td>291.8</td>\n",
       "      <td>28.1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_yap</th>\n",
       "      <td>435.0</td>\n",
       "      <td>64.0</td>\n",
       "      <td>384.0</td>\n",
       "      <td>40.0</td>\n",
       "      <td>289.2</td>\n",
       "      <td>28.3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>no_word</th>\n",
       "      <td>NaN</td>\n",
       "      <td>499.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>297.6</td>\n",
       "      <td>NaN</td>\n",
       "      <td>214.9</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                              true          pred         \\\n",
       "ooev                                         False  True   False  True    \n",
       "gold_name        unit  arch      w_embed                                  \n",
       "morph_dev_gold   morph char_cnn  ft_oov_tok  441.0   58.0  399.8   56.3   \n",
       "                                 ft_oov_yap  434.0   65.0  385.9   63.3   \n",
       "                                 ft_tok      441.0   58.0  395.9   50.3   \n",
       "                                 ft_yap      434.0   65.0  379.4   50.1   \n",
       "                                 glv_tok     441.0   58.0  391.8   50.0   \n",
       "                                 glv_yap     434.0   65.0  386.6   54.4   \n",
       "                                 no_word       NaN  499.0    NaN  331.4   \n",
       "                       char_lstm ft_oov_tok  441.0   58.0  404.5   55.9   \n",
       "                                 ft_oov_yap  434.0   65.0  383.6   62.4   \n",
       "                                 ft_tok      441.0   58.0  400.2   48.4   \n",
       "                                 ft_yap      434.0   65.0  386.5   51.4   \n",
       "                                 glv_tok     441.0   58.0  402.2   51.2   \n",
       "                                 glv_yap     434.0   65.0  400.6   56.1   \n",
       "                                 no_word       NaN  499.0    NaN  336.5   \n",
       "                       no_char   ft_oov_tok  441.0   58.0  398.3   54.7   \n",
       "                                 ft_oov_yap  434.0   65.0  384.9   59.8   \n",
       "                                 ft_tok      441.0   58.0  400.1   43.0   \n",
       "                                 ft_yap      434.0   65.0  382.1   41.9   \n",
       "                                 glv_tok     441.0   58.0  399.0   41.8   \n",
       "                                 glv_yap     434.0   65.0  392.6   44.6   \n",
       "                                 no_word       NaN  499.0    NaN  308.9   \n",
       "morph_dev_pruned morph char_cnn  ft_oov_tok  440.0   59.0  395.3   55.3   \n",
       "                                 ft_oov_yap  434.0   65.0  381.6   61.7   \n",
       "                                 ft_tok      438.0   61.0  388.8   50.4   \n",
       "                                 ft_yap      432.0   67.0  373.1   50.1   \n",
       "                                 glv_tok     439.0   60.0  387.7   49.4   \n",
       "                                 glv_yap     436.0   63.0  386.0   52.8   \n",
       "                                 no_word       NaN  499.0    NaN  320.1   \n",
       "                       char_lstm ft_oov_tok  440.0   59.0  398.4   53.9   \n",
       "                                 ft_oov_yap  435.0   64.0  380.0   61.1   \n",
       "                                 ft_tok      441.0   58.0  398.0   47.1   \n",
       "                                 ft_yap      435.0   64.0  382.1   50.0   \n",
       "                                 glv_tok     440.0   59.0  396.1   50.3   \n",
       "                                 glv_yap     434.0   65.0  398.6   56.2   \n",
       "                                 no_word       NaN  499.0    NaN  330.0   \n",
       "                       no_char   ft_oov_tok  438.0   61.0  392.9   53.5   \n",
       "                                 ft_oov_yap  430.0   69.0  376.0   59.3   \n",
       "                                 ft_tok      441.0   58.0  391.3   41.9   \n",
       "                                 ft_yap      431.0   68.0  374.0   41.4   \n",
       "                                 glv_tok     440.0   59.0  391.7   39.9   \n",
       "                                 glv_yap     429.0   70.0  384.3   47.2   \n",
       "                                 no_word       NaN  499.0    NaN  293.1   \n",
       "morph_dev_yap    morph char_cnn  ft_oov_tok  437.0   62.0  387.0   53.2   \n",
       "                                 ft_oov_yap  435.0   64.0  376.6   58.1   \n",
       "                                 ft_tok      437.0   62.0  386.0   47.8   \n",
       "                                 ft_yap      435.0   64.0  373.6   44.2   \n",
       "                                 glv_tok     437.0   62.0  385.1   46.9   \n",
       "                                 glv_yap     435.0   64.0  381.3   51.0   \n",
       "                                 no_word       NaN  499.0    NaN  320.4   \n",
       "                       char_lstm ft_oov_tok  437.0   62.0  392.4   52.6   \n",
       "                                 ft_oov_yap  435.0   64.0  374.0   56.7   \n",
       "                                 ft_tok      437.0   62.0  392.7   45.4   \n",
       "                                 ft_yap      435.0   64.0  381.4   45.8   \n",
       "                                 glv_tok     437.0   62.0  394.5   48.6   \n",
       "                                 glv_yap     435.0   64.0  395.7   52.0   \n",
       "                                 no_word       NaN  499.0    NaN  325.4   \n",
       "                       no_char   ft_oov_tok  437.0   62.0  385.9   51.4   \n",
       "                                 ft_oov_yap  435.0   64.0  374.9   54.5   \n",
       "                                 ft_tok      437.0   62.0  389.3   40.2   \n",
       "                                 ft_yap      435.0   64.0  373.7   37.1   \n",
       "                                 glv_tok     437.0   62.0  390.3   37.6   \n",
       "                                 glv_yap     435.0   64.0  384.0   40.0   \n",
       "                                 no_word       NaN  499.0    NaN  297.6   \n",
       "\n",
       "                                            correct         \n",
       "ooev                                          False  True   \n",
       "gold_name        unit  arch      w_embed                    \n",
       "morph_dev_gold   morph char_cnn  ft_oov_tok   337.7   43.0  \n",
       "                                 ft_oov_yap   328.0   51.4  \n",
       "                                 ft_tok       335.2   40.0  \n",
       "                                 ft_yap       324.7   41.7  \n",
       "                                 glv_tok      325.6   41.0  \n",
       "                                 glv_yap      321.1   45.6  \n",
       "                                 no_word        NaN  261.7  \n",
       "                       char_lstm ft_oov_tok   338.1   42.8  \n",
       "                                 ft_oov_yap   326.1   50.5  \n",
       "                                 ft_tok       334.6   38.8  \n",
       "                                 ft_yap       326.6   42.2  \n",
       "                                 glv_tok      329.0   41.8  \n",
       "                                 glv_yap      328.5   47.2  \n",
       "                                 no_word        NaN  262.7  \n",
       "                       no_char   ft_oov_tok   337.7   41.6  \n",
       "                                 ft_oov_yap   327.9   48.0  \n",
       "                                 ft_tok       335.9   34.1  \n",
       "                                 ft_yap       325.7   34.1  \n",
       "                                 glv_tok      325.1   33.9  \n",
       "                                 glv_yap      321.4   36.4  \n",
       "                                 no_word        NaN  241.5  \n",
       "morph_dev_pruned morph char_cnn  ft_oov_tok   326.2   42.0  \n",
       "                                 ft_oov_yap   318.6   49.5  \n",
       "                                 ft_tok       318.0   39.6  \n",
       "                                 ft_yap       314.3   40.2  \n",
       "                                 glv_tok      312.1   39.8  \n",
       "                                 glv_yap      313.2   43.2  \n",
       "                                 no_word        NaN  242.2  \n",
       "                       char_lstm ft_oov_tok   325.0   40.7  \n",
       "                                 ft_oov_yap   318.4   48.6  \n",
       "                                 ft_tok       319.6   37.0  \n",
       "                                 ft_yap       316.9   40.0  \n",
       "                                 glv_tok      315.3   38.6  \n",
       "                                 glv_yap      320.1   44.1  \n",
       "                                 no_word        NaN  245.3  \n",
       "                       no_char   ft_oov_tok   325.3   40.9  \n",
       "                                 ft_oov_yap   314.2   45.1  \n",
       "                                 ft_tok       318.6   33.0  \n",
       "                                 ft_yap       316.0   32.6  \n",
       "                                 glv_tok      311.6   32.3  \n",
       "                                 glv_yap      309.1   34.6  \n",
       "                                 no_word        NaN  219.3  \n",
       "morph_dev_yap    morph char_cnn  ft_oov_tok   302.8   36.9  \n",
       "                                 ft_oov_yap   295.5   43.4  \n",
       "                                 ft_tok       299.9   34.0  \n",
       "                                 ft_yap       291.6   34.3  \n",
       "                                 glv_tok      292.3   34.2  \n",
       "                                 glv_yap      289.2   36.6  \n",
       "                                 no_word        NaN  232.7  \n",
       "                       char_lstm ft_oov_tok   302.0   36.5  \n",
       "                                 ft_oov_yap   293.4   42.5  \n",
       "                                 ft_tok       298.9   33.0  \n",
       "                                 ft_yap       292.4   34.3  \n",
       "                                 glv_tok      294.4   35.0  \n",
       "                                 glv_yap      296.6   36.9  \n",
       "                                 no_word        NaN  235.2  \n",
       "                       no_char   ft_oov_tok   301.2   35.6  \n",
       "                                 ft_oov_yap   296.0   40.5  \n",
       "                                 ft_tok       300.0   29.1  \n",
       "                                 ft_yap       293.4   27.2  \n",
       "                                 glv_tok      291.8   28.1  \n",
       "                                 glv_yap      289.2   28.3  \n",
       "                                 no_word        NaN  214.9  "
      ]
     },
     "execution_count": 152,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(gsc[gsc.gold_name.str.contains('morph_dev')]\n",
    " .groupby(['gold_name', 'unit', 'arch', 'w_embed', 'ooev'], sort=False) \n",
    " [['true', 'pred', 'correct']].mean().unstack()).sort_index().round(1)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 153,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>ooev</th>\n",
       "      <th>False</th>\n",
       "      <th>True</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>gold_name</th>\n",
       "      <th>unit</th>\n",
       "      <th>arch</th>\n",
       "      <th>w_embed</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"42\" valign=\"top\">token_dev</th>\n",
       "      <th rowspan=\"21\" valign=\"top\">multitok</th>\n",
       "      <th rowspan=\"7\" valign=\"top\">char_cnn</th>\n",
       "      <th>ft_oov_tok</th>\n",
       "      <td>0.773465</td>\n",
       "      <td>0.741613</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_oov_yap</th>\n",
       "      <td>0.781569</td>\n",
       "      <td>0.681281</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_tok</th>\n",
       "      <td>0.768752</td>\n",
       "      <td>0.612234</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_yap</th>\n",
       "      <td>0.773227</td>\n",
       "      <td>0.562520</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_tok</th>\n",
       "      <td>0.740920</td>\n",
       "      <td>0.627760</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_yap</th>\n",
       "      <td>0.752973</td>\n",
       "      <td>0.572257</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>no_word</th>\n",
       "      <td>NaN</td>\n",
       "      <td>0.550150</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"7\" valign=\"top\">char_lstm</th>\n",
       "      <th>ft_oov_tok</th>\n",
       "      <td>0.775198</td>\n",
       "      <td>0.775788</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_oov_yap</th>\n",
       "      <td>0.781631</td>\n",
       "      <td>0.697106</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_tok</th>\n",
       "      <td>0.770221</td>\n",
       "      <td>0.628964</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_yap</th>\n",
       "      <td>0.781301</td>\n",
       "      <td>0.561815</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_tok</th>\n",
       "      <td>0.742581</td>\n",
       "      <td>0.627882</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_yap</th>\n",
       "      <td>0.746166</td>\n",
       "      <td>0.561709</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>no_word</th>\n",
       "      <td>NaN</td>\n",
       "      <td>0.555078</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"7\" valign=\"top\">no_char</th>\n",
       "      <th>ft_oov_tok</th>\n",
       "      <td>0.784532</td>\n",
       "      <td>0.760158</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_oov_yap</th>\n",
       "      <td>0.761864</td>\n",
       "      <td>0.691422</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_tok</th>\n",
       "      <td>0.777844</td>\n",
       "      <td>0.547463</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_yap</th>\n",
       "      <td>0.757022</td>\n",
       "      <td>0.504673</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_tok</th>\n",
       "      <td>0.732084</td>\n",
       "      <td>0.516589</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_yap</th>\n",
       "      <td>0.718755</td>\n",
       "      <td>0.502185</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>no_word</th>\n",
       "      <td>NaN</td>\n",
       "      <td>0.487085</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"21\" valign=\"top\">token</th>\n",
       "      <th rowspan=\"7\" valign=\"top\">char_cnn</th>\n",
       "      <th>ft_oov_tok</th>\n",
       "      <td>0.786543</td>\n",
       "      <td>0.767509</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_oov_yap</th>\n",
       "      <td>0.792602</td>\n",
       "      <td>0.694606</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_tok</th>\n",
       "      <td>0.773597</td>\n",
       "      <td>0.655330</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_yap</th>\n",
       "      <td>0.769943</td>\n",
       "      <td>0.552841</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_tok</th>\n",
       "      <td>0.750528</td>\n",
       "      <td>0.683003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_yap</th>\n",
       "      <td>0.751140</td>\n",
       "      <td>0.549205</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>no_word</th>\n",
       "      <td>NaN</td>\n",
       "      <td>0.563771</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"7\" valign=\"top\">char_lstm</th>\n",
       "      <th>ft_oov_tok</th>\n",
       "      <td>0.784439</td>\n",
       "      <td>0.763648</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_oov_yap</th>\n",
       "      <td>0.789244</td>\n",
       "      <td>0.692630</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_tok</th>\n",
       "      <td>0.781560</td>\n",
       "      <td>0.677570</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_yap</th>\n",
       "      <td>0.776668</td>\n",
       "      <td>0.546738</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_tok</th>\n",
       "      <td>0.749766</td>\n",
       "      <td>0.688014</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_yap</th>\n",
       "      <td>0.753654</td>\n",
       "      <td>0.559651</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>no_word</th>\n",
       "      <td>NaN</td>\n",
       "      <td>0.564065</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"7\" valign=\"top\">no_char</th>\n",
       "      <th>ft_oov_tok</th>\n",
       "      <td>0.781285</td>\n",
       "      <td>0.729724</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_oov_yap</th>\n",
       "      <td>0.786229</td>\n",
       "      <td>0.671648</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_tok</th>\n",
       "      <td>0.772143</td>\n",
       "      <td>0.569550</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_yap</th>\n",
       "      <td>0.766293</td>\n",
       "      <td>0.515855</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_tok</th>\n",
       "      <td>0.735932</td>\n",
       "      <td>0.539780</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_yap</th>\n",
       "      <td>0.728861</td>\n",
       "      <td>0.511592</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>no_word</th>\n",
       "      <td>NaN</td>\n",
       "      <td>0.528494</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "ooev                                        False     True \n",
       "gold_name unit     arch      w_embed                       \n",
       "token_dev multitok char_cnn  ft_oov_tok  0.773465  0.741613\n",
       "                             ft_oov_yap  0.781569  0.681281\n",
       "                             ft_tok      0.768752  0.612234\n",
       "                             ft_yap      0.773227  0.562520\n",
       "                             glv_tok     0.740920  0.627760\n",
       "                             glv_yap     0.752973  0.572257\n",
       "                             no_word          NaN  0.550150\n",
       "                   char_lstm ft_oov_tok  0.775198  0.775788\n",
       "                             ft_oov_yap  0.781631  0.697106\n",
       "                             ft_tok      0.770221  0.628964\n",
       "                             ft_yap      0.781301  0.561815\n",
       "                             glv_tok     0.742581  0.627882\n",
       "                             glv_yap     0.746166  0.561709\n",
       "                             no_word          NaN  0.555078\n",
       "                   no_char   ft_oov_tok  0.784532  0.760158\n",
       "                             ft_oov_yap  0.761864  0.691422\n",
       "                             ft_tok      0.777844  0.547463\n",
       "                             ft_yap      0.757022  0.504673\n",
       "                             glv_tok     0.732084  0.516589\n",
       "                             glv_yap     0.718755  0.502185\n",
       "                             no_word          NaN  0.487085\n",
       "          token    char_cnn  ft_oov_tok  0.786543  0.767509\n",
       "                             ft_oov_yap  0.792602  0.694606\n",
       "                             ft_tok      0.773597  0.655330\n",
       "                             ft_yap      0.769943  0.552841\n",
       "                             glv_tok     0.750528  0.683003\n",
       "                             glv_yap     0.751140  0.549205\n",
       "                             no_word          NaN  0.563771\n",
       "                   char_lstm ft_oov_tok  0.784439  0.763648\n",
       "                             ft_oov_yap  0.789244  0.692630\n",
       "                             ft_tok      0.781560  0.677570\n",
       "                             ft_yap      0.776668  0.546738\n",
       "                             glv_tok     0.749766  0.688014\n",
       "                             glv_yap     0.753654  0.559651\n",
       "                             no_word          NaN  0.564065\n",
       "                   no_char   ft_oov_tok  0.781285  0.729724\n",
       "                             ft_oov_yap  0.786229  0.671648\n",
       "                             ft_tok      0.772143  0.569550\n",
       "                             ft_yap      0.766293  0.515855\n",
       "                             glv_tok     0.735932  0.539780\n",
       "                             glv_yap     0.728861  0.511592\n",
       "                             no_word          NaN  0.528494"
      ]
     },
     "execution_count": 153,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(gsc[gsc.gold_name.str.contains('token_dev')]\n",
    " .groupby(['gold_name', 'unit', 'arch', 'w_embed', 'ooev'], sort=False)\n",
    " .f.mean().sort_index().unstack().sort_index())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 154,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>ooev</th>\n",
       "      <th>False</th>\n",
       "      <th>True</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>gold_name</th>\n",
       "      <th>unit</th>\n",
       "      <th>arch</th>\n",
       "      <th>w_embed</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"21\" valign=\"top\">morph_dev_gold</th>\n",
       "      <th rowspan=\"21\" valign=\"top\">morph</th>\n",
       "      <th rowspan=\"7\" valign=\"top\">char_cnn</th>\n",
       "      <th>ft_oov_tok</th>\n",
       "      <td>80.33</td>\n",
       "      <td>75.25</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_oov_yap</th>\n",
       "      <td>80.01</td>\n",
       "      <td>80.12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_tok</th>\n",
       "      <td>80.11</td>\n",
       "      <td>73.83</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_yap</th>\n",
       "      <td>79.84</td>\n",
       "      <td>72.43</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_tok</th>\n",
       "      <td>78.19</td>\n",
       "      <td>75.88</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_yap</th>\n",
       "      <td>78.26</td>\n",
       "      <td>76.34</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>no_word</th>\n",
       "      <td>NaN</td>\n",
       "      <td>63.03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"7\" valign=\"top\">char_lstm</th>\n",
       "      <th>ft_oov_tok</th>\n",
       "      <td>79.98</td>\n",
       "      <td>75.15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_oov_yap</th>\n",
       "      <td>79.77</td>\n",
       "      <td>79.28</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_tok</th>\n",
       "      <td>79.56</td>\n",
       "      <td>72.90</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_yap</th>\n",
       "      <td>79.61</td>\n",
       "      <td>72.49</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_tok</th>\n",
       "      <td>78.04</td>\n",
       "      <td>76.55</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_yap</th>\n",
       "      <td>78.72</td>\n",
       "      <td>77.96</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>no_word</th>\n",
       "      <td>NaN</td>\n",
       "      <td>62.89</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"7\" valign=\"top\">no_char</th>\n",
       "      <th>ft_oov_tok</th>\n",
       "      <td>80.47</td>\n",
       "      <td>73.82</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_oov_yap</th>\n",
       "      <td>80.08</td>\n",
       "      <td>76.92</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_tok</th>\n",
       "      <td>79.87</td>\n",
       "      <td>67.56</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_yap</th>\n",
       "      <td>79.82</td>\n",
       "      <td>63.79</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_tok</th>\n",
       "      <td>77.41</td>\n",
       "      <td>67.92</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_yap</th>\n",
       "      <td>77.76</td>\n",
       "      <td>66.37</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>no_word</th>\n",
       "      <td>NaN</td>\n",
       "      <td>59.78</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"21\" valign=\"top\">morph_dev_pruned</th>\n",
       "      <th rowspan=\"21\" valign=\"top\">morph</th>\n",
       "      <th rowspan=\"7\" valign=\"top\">char_cnn</th>\n",
       "      <th>ft_oov_tok</th>\n",
       "      <td>78.11</td>\n",
       "      <td>73.50</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_oov_yap</th>\n",
       "      <td>78.13</td>\n",
       "      <td>78.14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_tok</th>\n",
       "      <td>76.93</td>\n",
       "      <td>71.06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_yap</th>\n",
       "      <td>78.08</td>\n",
       "      <td>68.62</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_tok</th>\n",
       "      <td>75.51</td>\n",
       "      <td>72.72</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_yap</th>\n",
       "      <td>76.21</td>\n",
       "      <td>74.59</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>no_word</th>\n",
       "      <td>NaN</td>\n",
       "      <td>59.14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"7\" valign=\"top\">char_lstm</th>\n",
       "      <th>ft_oov_tok</th>\n",
       "      <td>77.53</td>\n",
       "      <td>72.10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_oov_yap</th>\n",
       "      <td>78.13</td>\n",
       "      <td>77.70</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_tok</th>\n",
       "      <td>76.19</td>\n",
       "      <td>70.40</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_yap</th>\n",
       "      <td>77.57</td>\n",
       "      <td>70.15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_tok</th>\n",
       "      <td>75.42</td>\n",
       "      <td>70.61</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_yap</th>\n",
       "      <td>76.90</td>\n",
       "      <td>72.80</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>no_word</th>\n",
       "      <td>NaN</td>\n",
       "      <td>59.19</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"7\" valign=\"top\">no_char</th>\n",
       "      <th>ft_oov_tok</th>\n",
       "      <td>78.30</td>\n",
       "      <td>71.44</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_oov_yap</th>\n",
       "      <td>77.97</td>\n",
       "      <td>70.30</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_tok</th>\n",
       "      <td>76.56</td>\n",
       "      <td>66.07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_yap</th>\n",
       "      <td>78.51</td>\n",
       "      <td>59.57</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_tok</th>\n",
       "      <td>74.93</td>\n",
       "      <td>65.29</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_yap</th>\n",
       "      <td>76.01</td>\n",
       "      <td>59.07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>no_word</th>\n",
       "      <td>NaN</td>\n",
       "      <td>55.37</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"21\" valign=\"top\">morph_dev_yap</th>\n",
       "      <th rowspan=\"21\" valign=\"top\">morph</th>\n",
       "      <th rowspan=\"7\" valign=\"top\">char_cnn</th>\n",
       "      <th>ft_oov_tok</th>\n",
       "      <td>73.50</td>\n",
       "      <td>64.07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_oov_yap</th>\n",
       "      <td>72.82</td>\n",
       "      <td>71.10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_tok</th>\n",
       "      <td>72.88</td>\n",
       "      <td>61.92</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_yap</th>\n",
       "      <td>72.13</td>\n",
       "      <td>63.37</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_tok</th>\n",
       "      <td>71.11</td>\n",
       "      <td>62.76</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_yap</th>\n",
       "      <td>70.86</td>\n",
       "      <td>63.66</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>no_word</th>\n",
       "      <td>NaN</td>\n",
       "      <td>56.80</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"7\" valign=\"top\">char_lstm</th>\n",
       "      <th>ft_oov_tok</th>\n",
       "      <td>72.83</td>\n",
       "      <td>63.70</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_oov_yap</th>\n",
       "      <td>72.53</td>\n",
       "      <td>70.42</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_tok</th>\n",
       "      <td>72.06</td>\n",
       "      <td>61.43</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_yap</th>\n",
       "      <td>71.63</td>\n",
       "      <td>62.49</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_tok</th>\n",
       "      <td>70.82</td>\n",
       "      <td>63.30</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_yap</th>\n",
       "      <td>71.41</td>\n",
       "      <td>63.64</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>no_word</th>\n",
       "      <td>NaN</td>\n",
       "      <td>57.07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"7\" valign=\"top\">no_char</th>\n",
       "      <th>ft_oov_tok</th>\n",
       "      <td>73.21</td>\n",
       "      <td>62.78</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_oov_yap</th>\n",
       "      <td>73.09</td>\n",
       "      <td>68.35</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_tok</th>\n",
       "      <td>72.61</td>\n",
       "      <td>56.92</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_yap</th>\n",
       "      <td>72.56</td>\n",
       "      <td>53.79</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_tok</th>\n",
       "      <td>70.54</td>\n",
       "      <td>56.39</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_yap</th>\n",
       "      <td>70.62</td>\n",
       "      <td>54.35</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>no_word</th>\n",
       "      <td>NaN</td>\n",
       "      <td>53.95</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"42\" valign=\"top\">token_dev</th>\n",
       "      <th rowspan=\"21\" valign=\"top\">multitok</th>\n",
       "      <th rowspan=\"7\" valign=\"top\">char_cnn</th>\n",
       "      <th>ft_oov_tok</th>\n",
       "      <td>77.35</td>\n",
       "      <td>74.16</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_oov_yap</th>\n",
       "      <td>78.16</td>\n",
       "      <td>68.13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_tok</th>\n",
       "      <td>76.88</td>\n",
       "      <td>61.22</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_yap</th>\n",
       "      <td>77.32</td>\n",
       "      <td>56.25</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_tok</th>\n",
       "      <td>74.09</td>\n",
       "      <td>62.78</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_yap</th>\n",
       "      <td>75.30</td>\n",
       "      <td>57.23</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>no_word</th>\n",
       "      <td>NaN</td>\n",
       "      <td>55.02</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"7\" valign=\"top\">char_lstm</th>\n",
       "      <th>ft_oov_tok</th>\n",
       "      <td>77.52</td>\n",
       "      <td>77.58</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_oov_yap</th>\n",
       "      <td>78.16</td>\n",
       "      <td>69.71</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_tok</th>\n",
       "      <td>77.02</td>\n",
       "      <td>62.90</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_yap</th>\n",
       "      <td>78.13</td>\n",
       "      <td>56.18</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_tok</th>\n",
       "      <td>74.26</td>\n",
       "      <td>62.79</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_yap</th>\n",
       "      <td>74.62</td>\n",
       "      <td>56.17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>no_word</th>\n",
       "      <td>NaN</td>\n",
       "      <td>55.51</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"7\" valign=\"top\">no_char</th>\n",
       "      <th>ft_oov_tok</th>\n",
       "      <td>78.45</td>\n",
       "      <td>76.02</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_oov_yap</th>\n",
       "      <td>76.19</td>\n",
       "      <td>69.14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_tok</th>\n",
       "      <td>77.78</td>\n",
       "      <td>54.75</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_yap</th>\n",
       "      <td>75.70</td>\n",
       "      <td>50.47</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_tok</th>\n",
       "      <td>73.21</td>\n",
       "      <td>51.66</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_yap</th>\n",
       "      <td>71.88</td>\n",
       "      <td>50.22</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>no_word</th>\n",
       "      <td>NaN</td>\n",
       "      <td>48.71</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"21\" valign=\"top\">token</th>\n",
       "      <th rowspan=\"7\" valign=\"top\">char_cnn</th>\n",
       "      <th>ft_oov_tok</th>\n",
       "      <td>78.65</td>\n",
       "      <td>76.75</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_oov_yap</th>\n",
       "      <td>79.26</td>\n",
       "      <td>69.46</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_tok</th>\n",
       "      <td>77.36</td>\n",
       "      <td>65.53</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_yap</th>\n",
       "      <td>76.99</td>\n",
       "      <td>55.28</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_tok</th>\n",
       "      <td>75.05</td>\n",
       "      <td>68.30</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_yap</th>\n",
       "      <td>75.11</td>\n",
       "      <td>54.92</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>no_word</th>\n",
       "      <td>NaN</td>\n",
       "      <td>56.38</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"7\" valign=\"top\">char_lstm</th>\n",
       "      <th>ft_oov_tok</th>\n",
       "      <td>78.44</td>\n",
       "      <td>76.36</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_oov_yap</th>\n",
       "      <td>78.92</td>\n",
       "      <td>69.26</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_tok</th>\n",
       "      <td>78.16</td>\n",
       "      <td>67.76</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_yap</th>\n",
       "      <td>77.67</td>\n",
       "      <td>54.67</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_tok</th>\n",
       "      <td>74.98</td>\n",
       "      <td>68.80</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_yap</th>\n",
       "      <td>75.37</td>\n",
       "      <td>55.97</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>no_word</th>\n",
       "      <td>NaN</td>\n",
       "      <td>56.41</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"7\" valign=\"top\">no_char</th>\n",
       "      <th>ft_oov_tok</th>\n",
       "      <td>78.13</td>\n",
       "      <td>72.97</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_oov_yap</th>\n",
       "      <td>78.62</td>\n",
       "      <td>67.16</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_tok</th>\n",
       "      <td>77.21</td>\n",
       "      <td>56.96</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_yap</th>\n",
       "      <td>76.63</td>\n",
       "      <td>51.59</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_tok</th>\n",
       "      <td>73.59</td>\n",
       "      <td>53.98</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_yap</th>\n",
       "      <td>72.89</td>\n",
       "      <td>51.16</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>no_word</th>\n",
       "      <td>NaN</td>\n",
       "      <td>52.85</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "ooev                                            False  True \n",
       "gold_name        unit     arch      w_embed                 \n",
       "morph_dev_gold   morph    char_cnn  ft_oov_tok  80.33  75.25\n",
       "                                    ft_oov_yap  80.01  80.12\n",
       "                                    ft_tok      80.11  73.83\n",
       "                                    ft_yap      79.84  72.43\n",
       "                                    glv_tok     78.19  75.88\n",
       "                                    glv_yap     78.26  76.34\n",
       "                                    no_word       NaN  63.03\n",
       "                          char_lstm ft_oov_tok  79.98  75.15\n",
       "                                    ft_oov_yap  79.77  79.28\n",
       "                                    ft_tok      79.56  72.90\n",
       "                                    ft_yap      79.61  72.49\n",
       "                                    glv_tok     78.04  76.55\n",
       "                                    glv_yap     78.72  77.96\n",
       "                                    no_word       NaN  62.89\n",
       "                          no_char   ft_oov_tok  80.47  73.82\n",
       "                                    ft_oov_yap  80.08  76.92\n",
       "                                    ft_tok      79.87  67.56\n",
       "                                    ft_yap      79.82  63.79\n",
       "                                    glv_tok     77.41  67.92\n",
       "                                    glv_yap     77.76  66.37\n",
       "                                    no_word       NaN  59.78\n",
       "morph_dev_pruned morph    char_cnn  ft_oov_tok  78.11  73.50\n",
       "                                    ft_oov_yap  78.13  78.14\n",
       "                                    ft_tok      76.93  71.06\n",
       "                                    ft_yap      78.08  68.62\n",
       "                                    glv_tok     75.51  72.72\n",
       "                                    glv_yap     76.21  74.59\n",
       "                                    no_word       NaN  59.14\n",
       "                          char_lstm ft_oov_tok  77.53  72.10\n",
       "                                    ft_oov_yap  78.13  77.70\n",
       "                                    ft_tok      76.19  70.40\n",
       "                                    ft_yap      77.57  70.15\n",
       "                                    glv_tok     75.42  70.61\n",
       "                                    glv_yap     76.90  72.80\n",
       "                                    no_word       NaN  59.19\n",
       "                          no_char   ft_oov_tok  78.30  71.44\n",
       "                                    ft_oov_yap  77.97  70.30\n",
       "                                    ft_tok      76.56  66.07\n",
       "                                    ft_yap      78.51  59.57\n",
       "                                    glv_tok     74.93  65.29\n",
       "                                    glv_yap     76.01  59.07\n",
       "                                    no_word       NaN  55.37\n",
       "morph_dev_yap    morph    char_cnn  ft_oov_tok  73.50  64.07\n",
       "                                    ft_oov_yap  72.82  71.10\n",
       "                                    ft_tok      72.88  61.92\n",
       "                                    ft_yap      72.13  63.37\n",
       "                                    glv_tok     71.11  62.76\n",
       "                                    glv_yap     70.86  63.66\n",
       "                                    no_word       NaN  56.80\n",
       "                          char_lstm ft_oov_tok  72.83  63.70\n",
       "                                    ft_oov_yap  72.53  70.42\n",
       "                                    ft_tok      72.06  61.43\n",
       "                                    ft_yap      71.63  62.49\n",
       "                                    glv_tok     70.82  63.30\n",
       "                                    glv_yap     71.41  63.64\n",
       "                                    no_word       NaN  57.07\n",
       "                          no_char   ft_oov_tok  73.21  62.78\n",
       "                                    ft_oov_yap  73.09  68.35\n",
       "                                    ft_tok      72.61  56.92\n",
       "                                    ft_yap      72.56  53.79\n",
       "                                    glv_tok     70.54  56.39\n",
       "                                    glv_yap     70.62  54.35\n",
       "                                    no_word       NaN  53.95\n",
       "token_dev        multitok char_cnn  ft_oov_tok  77.35  74.16\n",
       "                                    ft_oov_yap  78.16  68.13\n",
       "                                    ft_tok      76.88  61.22\n",
       "                                    ft_yap      77.32  56.25\n",
       "                                    glv_tok     74.09  62.78\n",
       "                                    glv_yap     75.30  57.23\n",
       "                                    no_word       NaN  55.02\n",
       "                          char_lstm ft_oov_tok  77.52  77.58\n",
       "                                    ft_oov_yap  78.16  69.71\n",
       "                                    ft_tok      77.02  62.90\n",
       "                                    ft_yap      78.13  56.18\n",
       "                                    glv_tok     74.26  62.79\n",
       "                                    glv_yap     74.62  56.17\n",
       "                                    no_word       NaN  55.51\n",
       "                          no_char   ft_oov_tok  78.45  76.02\n",
       "                                    ft_oov_yap  76.19  69.14\n",
       "                                    ft_tok      77.78  54.75\n",
       "                                    ft_yap      75.70  50.47\n",
       "                                    glv_tok     73.21  51.66\n",
       "                                    glv_yap     71.88  50.22\n",
       "                                    no_word       NaN  48.71\n",
       "                 token    char_cnn  ft_oov_tok  78.65  76.75\n",
       "                                    ft_oov_yap  79.26  69.46\n",
       "                                    ft_tok      77.36  65.53\n",
       "                                    ft_yap      76.99  55.28\n",
       "                                    glv_tok     75.05  68.30\n",
       "                                    glv_yap     75.11  54.92\n",
       "                                    no_word       NaN  56.38\n",
       "                          char_lstm ft_oov_tok  78.44  76.36\n",
       "                                    ft_oov_yap  78.92  69.26\n",
       "                                    ft_tok      78.16  67.76\n",
       "                                    ft_yap      77.67  54.67\n",
       "                                    glv_tok     74.98  68.80\n",
       "                                    glv_yap     75.37  55.97\n",
       "                                    no_word       NaN  56.41\n",
       "                          no_char   ft_oov_tok  78.13  72.97\n",
       "                                    ft_oov_yap  78.62  67.16\n",
       "                                    ft_tok      77.21  56.96\n",
       "                                    ft_yap      76.63  51.59\n",
       "                                    glv_tok     73.59  53.98\n",
       "                                    glv_yap     72.89  51.16\n",
       "                                    no_word       NaN  52.85"
      ]
     },
     "execution_count": 154,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(gsc[gsc.gold_name.str.contains('dev')]\n",
    " .groupby(['gold_name', 'unit', 'arch', 'w_embed', 'ooev'], sort=False)\n",
    " .f.mean().mul(100).round(2).sort_index().unstack().sort_index())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## morpheme count\n",
    "GloVe is consistently better on tokens that were segmented into more morphemes. It is actually the only one better on 3-morpheme than 2-morpheme. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 595,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1    6078\n",
       "2    2143\n",
       "3     303\n",
       "4       7\n",
       "Name: morpheme_count, dtype: int64"
      ]
     },
     "execution_count": 595,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dev_unk.morpheme_count.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 596,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sent_id</th>\n",
       "      <th>token_id</th>\n",
       "      <th>token_str</th>\n",
       "      <th>token_unk</th>\n",
       "      <th>morph_unk</th>\n",
       "      <th>lemma_unk</th>\n",
       "      <th>unk_type</th>\n",
       "      <th>has_ner</th>\n",
       "      <th>morpheme_count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>583</th>\n",
       "      <td>27</td>\n",
       "      <td>22</td>\n",
       "      <td>שבמנזר</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>TM</td>\n",
       "      <td>False</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1581</th>\n",
       "      <td>74</td>\n",
       "      <td>3</td>\n",
       "      <td>ולאינפלציה</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>T</td>\n",
       "      <td>False</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1981</th>\n",
       "      <td>98</td>\n",
       "      <td>19</td>\n",
       "      <td>ושהאותוריטה</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>TM</td>\n",
       "      <td>False</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4853</th>\n",
       "      <td>259</td>\n",
       "      <td>24</td>\n",
       "      <td>שבעתיד</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>T</td>\n",
       "      <td>False</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5265</th>\n",
       "      <td>279</td>\n",
       "      <td>3</td>\n",
       "      <td>שבהצבעה</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>T</td>\n",
       "      <td>False</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6494</th>\n",
       "      <td>351</td>\n",
       "      <td>23</td>\n",
       "      <td>ובמקום</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>T</td>\n",
       "      <td>False</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8391</th>\n",
       "      <td>487</td>\n",
       "      <td>12</td>\n",
       "      <td>וברגליים</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>T</td>\n",
       "      <td>False</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      sent_id  token_id    token_str  token_unk  morph_unk  lemma_unk  \\\n",
       "583        27        22       שבמנזר       True       True       True   \n",
       "1581       74         3   ולאינפלציה       True      False      False   \n",
       "1981       98        19  ושהאותוריטה       True       True       True   \n",
       "4853      259        24       שבעתיד       True      False      False   \n",
       "5265      279         3      שבהצבעה       True      False      False   \n",
       "6494      351        23       ובמקום       True      False      False   \n",
       "8391      487        12     וברגליים       True      False      False   \n",
       "\n",
       "     unk_type  has_ner  morpheme_count  \n",
       "583        TM    False               4  \n",
       "1581        T    False               4  \n",
       "1981       TM    False               4  \n",
       "4853        T    False               4  \n",
       "5265        T    False               4  \n",
       "6494        T    False               4  \n",
       "8391        T    False               4  "
      ]
     },
     "execution_count": 596,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dev_unk[dev_unk.morpheme_count==4]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 159,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>morpheme_count</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>gold_name</th>\n",
       "      <th>unit</th>\n",
       "      <th>arch</th>\n",
       "      <th>w_embed</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"21\" valign=\"top\">morph_dev_gold</th>\n",
       "      <th rowspan=\"21\" valign=\"top\">morph</th>\n",
       "      <th rowspan=\"7\" valign=\"top\">char_cnn</th>\n",
       "      <th>ft_oov_tok</th>\n",
       "      <td>83.11</td>\n",
       "      <td>75.21</td>\n",
       "      <td>75.89</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_oov_yap</th>\n",
       "      <td>83.55</td>\n",
       "      <td>75.19</td>\n",
       "      <td>76.70</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_tok</th>\n",
       "      <td>82.74</td>\n",
       "      <td>74.77</td>\n",
       "      <td>76.48</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_yap</th>\n",
       "      <td>82.33</td>\n",
       "      <td>74.17</td>\n",
       "      <td>76.46</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_tok</th>\n",
       "      <td>81.18</td>\n",
       "      <td>72.79</td>\n",
       "      <td>78.35</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_yap</th>\n",
       "      <td>80.77</td>\n",
       "      <td>73.37</td>\n",
       "      <td>80.08</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>no_word</th>\n",
       "      <td>62.45</td>\n",
       "      <td>60.84</td>\n",
       "      <td>78.14</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"7\" valign=\"top\">char_lstm</th>\n",
       "      <th>ft_oov_tok</th>\n",
       "      <td>82.13</td>\n",
       "      <td>76.01</td>\n",
       "      <td>74.67</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_oov_yap</th>\n",
       "      <td>83.25</td>\n",
       "      <td>74.75</td>\n",
       "      <td>76.54</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_tok</th>\n",
       "      <td>81.77</td>\n",
       "      <td>74.74</td>\n",
       "      <td>76.19</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_yap</th>\n",
       "      <td>81.64</td>\n",
       "      <td>74.70</td>\n",
       "      <td>76.19</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_tok</th>\n",
       "      <td>81.19</td>\n",
       "      <td>72.86</td>\n",
       "      <td>77.03</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_yap</th>\n",
       "      <td>81.27</td>\n",
       "      <td>74.51</td>\n",
       "      <td>78.69</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>no_word</th>\n",
       "      <td>62.52</td>\n",
       "      <td>60.60</td>\n",
       "      <td>76.96</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"7\" valign=\"top\">no_char</th>\n",
       "      <th>ft_oov_tok</th>\n",
       "      <td>82.39</td>\n",
       "      <td>76.28</td>\n",
       "      <td>75.50</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_oov_yap</th>\n",
       "      <td>82.90</td>\n",
       "      <td>75.26</td>\n",
       "      <td>76.53</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_tok</th>\n",
       "      <td>81.13</td>\n",
       "      <td>75.41</td>\n",
       "      <td>74.27</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_yap</th>\n",
       "      <td>79.86</td>\n",
       "      <td>75.39</td>\n",
       "      <td>76.33</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_tok</th>\n",
       "      <td>78.96</td>\n",
       "      <td>72.20</td>\n",
       "      <td>77.93</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_yap</th>\n",
       "      <td>78.25</td>\n",
       "      <td>73.10</td>\n",
       "      <td>79.53</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>no_word</th>\n",
       "      <td>55.80</td>\n",
       "      <td>61.71</td>\n",
       "      <td>77.07</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"21\" valign=\"top\">morph_dev_pruned</th>\n",
       "      <th rowspan=\"21\" valign=\"top\">morph</th>\n",
       "      <th rowspan=\"7\" valign=\"top\">char_cnn</th>\n",
       "      <th>ft_oov_tok</th>\n",
       "      <td>81.32</td>\n",
       "      <td>72.58</td>\n",
       "      <td>72.45</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_oov_yap</th>\n",
       "      <td>82.46</td>\n",
       "      <td>73.12</td>\n",
       "      <td>68.33</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_tok</th>\n",
       "      <td>80.98</td>\n",
       "      <td>70.75</td>\n",
       "      <td>65.87</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_yap</th>\n",
       "      <td>81.52</td>\n",
       "      <td>71.58</td>\n",
       "      <td>66.96</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_tok</th>\n",
       "      <td>79.69</td>\n",
       "      <td>68.70</td>\n",
       "      <td>71.76</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_yap</th>\n",
       "      <td>79.32</td>\n",
       "      <td>71.11</td>\n",
       "      <td>74.63</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>no_word</th>\n",
       "      <td>60.53</td>\n",
       "      <td>55.31</td>\n",
       "      <td>68.04</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"7\" valign=\"top\">char_lstm</th>\n",
       "      <th>ft_oov_tok</th>\n",
       "      <td>80.19</td>\n",
       "      <td>73.64</td>\n",
       "      <td>66.07</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_oov_yap</th>\n",
       "      <td>82.44</td>\n",
       "      <td>72.68</td>\n",
       "      <td>70.33</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_tok</th>\n",
       "      <td>79.95</td>\n",
       "      <td>70.31</td>\n",
       "      <td>66.78</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_yap</th>\n",
       "      <td>80.07</td>\n",
       "      <td>72.62</td>\n",
       "      <td>70.10</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_tok</th>\n",
       "      <td>79.57</td>\n",
       "      <td>68.69</td>\n",
       "      <td>68.05</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_yap</th>\n",
       "      <td>79.45</td>\n",
       "      <td>72.00</td>\n",
       "      <td>73.95</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>no_word</th>\n",
       "      <td>60.25</td>\n",
       "      <td>56.50</td>\n",
       "      <td>64.77</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"7\" valign=\"top\">no_char</th>\n",
       "      <th>ft_oov_tok</th>\n",
       "      <td>80.74</td>\n",
       "      <td>73.72</td>\n",
       "      <td>70.03</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_oov_yap</th>\n",
       "      <td>81.96</td>\n",
       "      <td>69.98</td>\n",
       "      <td>70.97</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_tok</th>\n",
       "      <td>79.81</td>\n",
       "      <td>70.53</td>\n",
       "      <td>65.20</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_yap</th>\n",
       "      <td>79.87</td>\n",
       "      <td>72.06</td>\n",
       "      <td>68.90</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_tok</th>\n",
       "      <td>77.68</td>\n",
       "      <td>69.07</td>\n",
       "      <td>68.03</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_yap</th>\n",
       "      <td>77.25</td>\n",
       "      <td>68.82</td>\n",
       "      <td>73.05</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>no_word</th>\n",
       "      <td>54.83</td>\n",
       "      <td>55.13</td>\n",
       "      <td>60.38</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"21\" valign=\"top\">morph_dev_yap</th>\n",
       "      <th rowspan=\"21\" valign=\"top\">morph</th>\n",
       "      <th rowspan=\"7\" valign=\"top\">char_cnn</th>\n",
       "      <th>ft_oov_tok</th>\n",
       "      <td>78.48</td>\n",
       "      <td>63.44</td>\n",
       "      <td>66.39</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_oov_yap</th>\n",
       "      <td>78.86</td>\n",
       "      <td>63.67</td>\n",
       "      <td>66.18</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_tok</th>\n",
       "      <td>77.93</td>\n",
       "      <td>62.55</td>\n",
       "      <td>65.41</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_yap</th>\n",
       "      <td>77.37</td>\n",
       "      <td>62.23</td>\n",
       "      <td>65.46</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_tok</th>\n",
       "      <td>76.31</td>\n",
       "      <td>60.64</td>\n",
       "      <td>66.99</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_yap</th>\n",
       "      <td>75.90</td>\n",
       "      <td>60.21</td>\n",
       "      <td>70.80</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>no_word</th>\n",
       "      <td>59.73</td>\n",
       "      <td>50.18</td>\n",
       "      <td>67.94</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"7\" valign=\"top\">char_lstm</th>\n",
       "      <th>ft_oov_tok</th>\n",
       "      <td>77.07</td>\n",
       "      <td>64.07</td>\n",
       "      <td>65.49</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_oov_yap</th>\n",
       "      <td>78.48</td>\n",
       "      <td>63.21</td>\n",
       "      <td>66.38</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_tok</th>\n",
       "      <td>76.66</td>\n",
       "      <td>62.62</td>\n",
       "      <td>65.03</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_yap</th>\n",
       "      <td>76.38</td>\n",
       "      <td>62.17</td>\n",
       "      <td>65.61</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_tok</th>\n",
       "      <td>76.35</td>\n",
       "      <td>60.30</td>\n",
       "      <td>65.78</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_yap</th>\n",
       "      <td>76.40</td>\n",
       "      <td>60.95</td>\n",
       "      <td>70.38</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>no_word</th>\n",
       "      <td>59.41</td>\n",
       "      <td>51.61</td>\n",
       "      <td>66.53</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"7\" valign=\"top\">no_char</th>\n",
       "      <th>ft_oov_tok</th>\n",
       "      <td>77.34</td>\n",
       "      <td>64.25</td>\n",
       "      <td>65.97</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_oov_yap</th>\n",
       "      <td>78.50</td>\n",
       "      <td>63.90</td>\n",
       "      <td>66.67</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_tok</th>\n",
       "      <td>76.44</td>\n",
       "      <td>63.32</td>\n",
       "      <td>64.42</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_yap</th>\n",
       "      <td>75.34</td>\n",
       "      <td>63.88</td>\n",
       "      <td>64.90</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_tok</th>\n",
       "      <td>74.25</td>\n",
       "      <td>60.85</td>\n",
       "      <td>67.60</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_yap</th>\n",
       "      <td>73.87</td>\n",
       "      <td>60.59</td>\n",
       "      <td>69.64</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>no_word</th>\n",
       "      <td>53.39</td>\n",
       "      <td>51.98</td>\n",
       "      <td>67.22</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"42\" valign=\"top\">token_dev</th>\n",
       "      <th rowspan=\"21\" valign=\"top\">multitok</th>\n",
       "      <th rowspan=\"7\" valign=\"top\">char_cnn</th>\n",
       "      <th>ft_oov_tok</th>\n",
       "      <td>80.11</td>\n",
       "      <td>71.78</td>\n",
       "      <td>76.49</td>\n",
       "      <td>-100.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_oov_yap</th>\n",
       "      <td>78.50</td>\n",
       "      <td>68.32</td>\n",
       "      <td>77.75</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_tok</th>\n",
       "      <td>78.89</td>\n",
       "      <td>68.22</td>\n",
       "      <td>74.79</td>\n",
       "      <td>-100.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_yap</th>\n",
       "      <td>77.43</td>\n",
       "      <td>59.26</td>\n",
       "      <td>73.38</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_tok</th>\n",
       "      <td>76.98</td>\n",
       "      <td>65.50</td>\n",
       "      <td>71.88</td>\n",
       "      <td>-90.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_yap</th>\n",
       "      <td>75.88</td>\n",
       "      <td>58.14</td>\n",
       "      <td>73.54</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>no_word</th>\n",
       "      <td>57.22</td>\n",
       "      <td>49.66</td>\n",
       "      <td>64.28</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"7\" valign=\"top\">char_lstm</th>\n",
       "      <th>ft_oov_tok</th>\n",
       "      <td>80.93</td>\n",
       "      <td>72.32</td>\n",
       "      <td>76.02</td>\n",
       "      <td>-100.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_oov_yap</th>\n",
       "      <td>78.86</td>\n",
       "      <td>69.62</td>\n",
       "      <td>75.65</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_tok</th>\n",
       "      <td>78.76</td>\n",
       "      <td>69.21</td>\n",
       "      <td>74.75</td>\n",
       "      <td>-100.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_yap</th>\n",
       "      <td>78.04</td>\n",
       "      <td>60.02</td>\n",
       "      <td>72.32</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_tok</th>\n",
       "      <td>76.53</td>\n",
       "      <td>67.13</td>\n",
       "      <td>69.31</td>\n",
       "      <td>-90.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_yap</th>\n",
       "      <td>74.97</td>\n",
       "      <td>57.72</td>\n",
       "      <td>71.69</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>no_word</th>\n",
       "      <td>57.33</td>\n",
       "      <td>50.72</td>\n",
       "      <td>64.86</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"7\" valign=\"top\">no_char</th>\n",
       "      <th>ft_oov_tok</th>\n",
       "      <td>81.65</td>\n",
       "      <td>72.20</td>\n",
       "      <td>78.98</td>\n",
       "      <td>-100.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_oov_yap</th>\n",
       "      <td>77.96</td>\n",
       "      <td>67.16</td>\n",
       "      <td>74.23</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_tok</th>\n",
       "      <td>78.81</td>\n",
       "      <td>68.07</td>\n",
       "      <td>75.15</td>\n",
       "      <td>-100.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_yap</th>\n",
       "      <td>73.96</td>\n",
       "      <td>58.36</td>\n",
       "      <td>72.17</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_tok</th>\n",
       "      <td>75.18</td>\n",
       "      <td>63.05</td>\n",
       "      <td>69.76</td>\n",
       "      <td>-90.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_yap</th>\n",
       "      <td>71.11</td>\n",
       "      <td>53.88</td>\n",
       "      <td>73.77</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>no_word</th>\n",
       "      <td>49.96</td>\n",
       "      <td>45.50</td>\n",
       "      <td>55.07</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"21\" valign=\"top\">token</th>\n",
       "      <th rowspan=\"7\" valign=\"top\">char_cnn</th>\n",
       "      <th>ft_oov_tok</th>\n",
       "      <td>82.08</td>\n",
       "      <td>72.76</td>\n",
       "      <td>76.81</td>\n",
       "      <td>-100.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_oov_yap</th>\n",
       "      <td>80.79</td>\n",
       "      <td>68.50</td>\n",
       "      <td>75.16</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_tok</th>\n",
       "      <td>79.47</td>\n",
       "      <td>69.62</td>\n",
       "      <td>77.44</td>\n",
       "      <td>-100.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_yap</th>\n",
       "      <td>76.26</td>\n",
       "      <td>60.70</td>\n",
       "      <td>71.85</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_tok</th>\n",
       "      <td>80.45</td>\n",
       "      <td>64.25</td>\n",
       "      <td>72.08</td>\n",
       "      <td>-100.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_yap</th>\n",
       "      <td>74.70</td>\n",
       "      <td>58.61</td>\n",
       "      <td>72.27</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>no_word</th>\n",
       "      <td>57.15</td>\n",
       "      <td>52.35</td>\n",
       "      <td>69.99</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"7\" valign=\"top\">char_lstm</th>\n",
       "      <th>ft_oov_tok</th>\n",
       "      <td>81.96</td>\n",
       "      <td>72.25</td>\n",
       "      <td>76.87</td>\n",
       "      <td>-100.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_oov_yap</th>\n",
       "      <td>80.44</td>\n",
       "      <td>68.42</td>\n",
       "      <td>74.27</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_tok</th>\n",
       "      <td>81.11</td>\n",
       "      <td>70.29</td>\n",
       "      <td>74.13</td>\n",
       "      <td>-100.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_yap</th>\n",
       "      <td>76.34</td>\n",
       "      <td>61.34</td>\n",
       "      <td>71.21</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_tok</th>\n",
       "      <td>79.79</td>\n",
       "      <td>65.87</td>\n",
       "      <td>69.62</td>\n",
       "      <td>-90.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_yap</th>\n",
       "      <td>75.36</td>\n",
       "      <td>58.88</td>\n",
       "      <td>71.43</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>no_word</th>\n",
       "      <td>58.28</td>\n",
       "      <td>51.68</td>\n",
       "      <td>64.90</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"7\" valign=\"top\">no_char</th>\n",
       "      <th>ft_oov_tok</th>\n",
       "      <td>81.48</td>\n",
       "      <td>71.23</td>\n",
       "      <td>75.37</td>\n",
       "      <td>-100.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_oov_yap</th>\n",
       "      <td>79.98</td>\n",
       "      <td>66.55</td>\n",
       "      <td>75.36</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_tok</th>\n",
       "      <td>78.64</td>\n",
       "      <td>68.58</td>\n",
       "      <td>74.33</td>\n",
       "      <td>-100.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_yap</th>\n",
       "      <td>74.91</td>\n",
       "      <td>58.93</td>\n",
       "      <td>73.41</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_tok</th>\n",
       "      <td>77.14</td>\n",
       "      <td>61.73</td>\n",
       "      <td>70.38</td>\n",
       "      <td>-90.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_yap</th>\n",
       "      <td>72.03</td>\n",
       "      <td>55.45</td>\n",
       "      <td>73.26</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>no_word</th>\n",
       "      <td>52.39</td>\n",
       "      <td>50.63</td>\n",
       "      <td>66.70</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "morpheme_count                                      1      2      3      4\n",
       "gold_name        unit     arch      w_embed                               \n",
       "morph_dev_gold   morph    char_cnn  ft_oov_tok  83.11  75.21  75.89    0.0\n",
       "                                    ft_oov_yap  83.55  75.19  76.70    NaN\n",
       "                                    ft_tok      82.74  74.77  76.48    NaN\n",
       "                                    ft_yap      82.33  74.17  76.46    NaN\n",
       "                                    glv_tok     81.18  72.79  78.35    NaN\n",
       "                                    glv_yap     80.77  73.37  80.08    NaN\n",
       "                                    no_word     62.45  60.84  78.14    NaN\n",
       "                          char_lstm ft_oov_tok  82.13  76.01  74.67    NaN\n",
       "                                    ft_oov_yap  83.25  74.75  76.54    NaN\n",
       "                                    ft_tok      81.77  74.74  76.19    NaN\n",
       "                                    ft_yap      81.64  74.70  76.19    NaN\n",
       "                                    glv_tok     81.19  72.86  77.03    NaN\n",
       "                                    glv_yap     81.27  74.51  78.69    NaN\n",
       "                                    no_word     62.52  60.60  76.96    NaN\n",
       "                          no_char   ft_oov_tok  82.39  76.28  75.50    NaN\n",
       "                                    ft_oov_yap  82.90  75.26  76.53    NaN\n",
       "                                    ft_tok      81.13  75.41  74.27    NaN\n",
       "                                    ft_yap      79.86  75.39  76.33    NaN\n",
       "                                    glv_tok     78.96  72.20  77.93    NaN\n",
       "                                    glv_yap     78.25  73.10  79.53    NaN\n",
       "                                    no_word     55.80  61.71  77.07    NaN\n",
       "morph_dev_pruned morph    char_cnn  ft_oov_tok  81.32  72.58  72.45    NaN\n",
       "                                    ft_oov_yap  82.46  73.12  68.33    NaN\n",
       "                                    ft_tok      80.98  70.75  65.87    NaN\n",
       "                                    ft_yap      81.52  71.58  66.96    NaN\n",
       "                                    glv_tok     79.69  68.70  71.76    NaN\n",
       "                                    glv_yap     79.32  71.11  74.63    NaN\n",
       "                                    no_word     60.53  55.31  68.04    NaN\n",
       "                          char_lstm ft_oov_tok  80.19  73.64  66.07    NaN\n",
       "                                    ft_oov_yap  82.44  72.68  70.33    NaN\n",
       "                                    ft_tok      79.95  70.31  66.78    NaN\n",
       "                                    ft_yap      80.07  72.62  70.10    NaN\n",
       "                                    glv_tok     79.57  68.69  68.05    NaN\n",
       "                                    glv_yap     79.45  72.00  73.95    NaN\n",
       "                                    no_word     60.25  56.50  64.77    NaN\n",
       "                          no_char   ft_oov_tok  80.74  73.72  70.03    NaN\n",
       "                                    ft_oov_yap  81.96  69.98  70.97    NaN\n",
       "                                    ft_tok      79.81  70.53  65.20    NaN\n",
       "                                    ft_yap      79.87  72.06  68.90    NaN\n",
       "                                    glv_tok     77.68  69.07  68.03    NaN\n",
       "                                    glv_yap     77.25  68.82  73.05    NaN\n",
       "                                    no_word     54.83  55.13  60.38    NaN\n",
       "morph_dev_yap    morph    char_cnn  ft_oov_tok  78.48  63.44  66.39    0.0\n",
       "                                    ft_oov_yap  78.86  63.67  66.18    NaN\n",
       "                                    ft_tok      77.93  62.55  65.41    NaN\n",
       "                                    ft_yap      77.37  62.23  65.46    NaN\n",
       "                                    glv_tok     76.31  60.64  66.99    NaN\n",
       "                                    glv_yap     75.90  60.21  70.80    NaN\n",
       "                                    no_word     59.73  50.18  67.94    NaN\n",
       "                          char_lstm ft_oov_tok  77.07  64.07  65.49    NaN\n",
       "                                    ft_oov_yap  78.48  63.21  66.38    NaN\n",
       "                                    ft_tok      76.66  62.62  65.03    NaN\n",
       "                                    ft_yap      76.38  62.17  65.61    NaN\n",
       "                                    glv_tok     76.35  60.30  65.78    NaN\n",
       "                                    glv_yap     76.40  60.95  70.38    NaN\n",
       "                                    no_word     59.41  51.61  66.53    NaN\n",
       "                          no_char   ft_oov_tok  77.34  64.25  65.97    NaN\n",
       "                                    ft_oov_yap  78.50  63.90  66.67    NaN\n",
       "                                    ft_tok      76.44  63.32  64.42    NaN\n",
       "                                    ft_yap      75.34  63.88  64.90    NaN\n",
       "                                    glv_tok     74.25  60.85  67.60    NaN\n",
       "                                    glv_yap     73.87  60.59  69.64    NaN\n",
       "                                    no_word     53.39  51.98  67.22    NaN\n",
       "token_dev        multitok char_cnn  ft_oov_tok  80.11  71.78  76.49 -100.0\n",
       "                                    ft_oov_yap  78.50  68.32  77.75    NaN\n",
       "                                    ft_tok      78.89  68.22  74.79 -100.0\n",
       "                                    ft_yap      77.43  59.26  73.38    NaN\n",
       "                                    glv_tok     76.98  65.50  71.88  -90.0\n",
       "                                    glv_yap     75.88  58.14  73.54    NaN\n",
       "                                    no_word     57.22  49.66  64.28    NaN\n",
       "                          char_lstm ft_oov_tok  80.93  72.32  76.02 -100.0\n",
       "                                    ft_oov_yap  78.86  69.62  75.65    NaN\n",
       "                                    ft_tok      78.76  69.21  74.75 -100.0\n",
       "                                    ft_yap      78.04  60.02  72.32    NaN\n",
       "                                    glv_tok     76.53  67.13  69.31  -90.0\n",
       "                                    glv_yap     74.97  57.72  71.69    NaN\n",
       "                                    no_word     57.33  50.72  64.86    NaN\n",
       "                          no_char   ft_oov_tok  81.65  72.20  78.98 -100.0\n",
       "                                    ft_oov_yap  77.96  67.16  74.23    NaN\n",
       "                                    ft_tok      78.81  68.07  75.15 -100.0\n",
       "                                    ft_yap      73.96  58.36  72.17    NaN\n",
       "                                    glv_tok     75.18  63.05  69.76  -90.0\n",
       "                                    glv_yap     71.11  53.88  73.77    NaN\n",
       "                                    no_word     49.96  45.50  55.07    NaN\n",
       "                 token    char_cnn  ft_oov_tok  82.08  72.76  76.81 -100.0\n",
       "                                    ft_oov_yap  80.79  68.50  75.16    NaN\n",
       "                                    ft_tok      79.47  69.62  77.44 -100.0\n",
       "                                    ft_yap      76.26  60.70  71.85    NaN\n",
       "                                    glv_tok     80.45  64.25  72.08 -100.0\n",
       "                                    glv_yap     74.70  58.61  72.27    NaN\n",
       "                                    no_word     57.15  52.35  69.99    NaN\n",
       "                          char_lstm ft_oov_tok  81.96  72.25  76.87 -100.0\n",
       "                                    ft_oov_yap  80.44  68.42  74.27    NaN\n",
       "                                    ft_tok      81.11  70.29  74.13 -100.0\n",
       "                                    ft_yap      76.34  61.34  71.21    NaN\n",
       "                                    glv_tok     79.79  65.87  69.62  -90.0\n",
       "                                    glv_yap     75.36  58.88  71.43    NaN\n",
       "                                    no_word     58.28  51.68  64.90    NaN\n",
       "                          no_char   ft_oov_tok  81.48  71.23  75.37 -100.0\n",
       "                                    ft_oov_yap  79.98  66.55  75.36    NaN\n",
       "                                    ft_tok      78.64  68.58  74.33 -100.0\n",
       "                                    ft_yap      74.91  58.93  73.41    NaN\n",
       "                                    glv_tok     77.14  61.73  70.38  -90.0\n",
       "                                    glv_yap     72.03  55.45  73.26    NaN\n",
       "                                    no_word     52.39  50.63  66.70    NaN"
      ]
     },
     "execution_count": 159,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gsc = group_scores(scores5, groupers=[ignore, max, ignore, ignore], ment_grouper=ignore)\n",
    "g = (gsc[gsc.gold_name.str.contains('dev')].fillna('x')\n",
    " .groupby(['gold_name', 'unit', 'arch', 'w_embed', 'morpheme_count'], sort=False)\n",
    " .f.mean().mul(100).round(2).unstack()).sort_index()\n",
    "g.reindex(sorted(g.columns), axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 315,
   "metadata": {},
   "outputs": [],
   "source": [
    "gsc = group_scores(scores5, groupers=[ignore, lambda x: min(max(x), 2), ignore, ignore], ment_grouper=ignore, ment_cat_grouper=ignore)\n",
    "gsc['cm'] = gsc.apply(get_cm, axis=1)\n",
    "gsc['embed_type'] = gsc.w_embed.str.replace('_tok|_yap', '')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 317,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr:last-of-type th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>morpheme_count</th>\n",
       "      <th colspan=\"3\" halign=\"left\">1</th>\n",
       "      <th colspan=\"3\" halign=\"left\">2</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>embed_type</th>\n",
       "      <th>glv</th>\n",
       "      <th>ft</th>\n",
       "      <th>ft_oov</th>\n",
       "      <th>glv</th>\n",
       "      <th>ft</th>\n",
       "      <th>ft_oov</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>gold_name</th>\n",
       "      <th>unit</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>morph_dev_gold</th>\n",
       "      <th>morph</th>\n",
       "      <td>$80.77 ± 0.8$</td>\n",
       "      <td>$82.33 ± 0.7$</td>\n",
       "      <td>$83.55 ± 1.2$</td>\n",
       "      <td>$74.37 ± 0.7$</td>\n",
       "      <td>$74.53 ± 0.6$</td>\n",
       "      <td>$75.43 ± 0.5$</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>morph_dev_pruned</th>\n",
       "      <th>morph</th>\n",
       "      <td>$79.32 ± 0.9$</td>\n",
       "      <td>$81.52 ± 0.5$</td>\n",
       "      <td>$82.46 ± 1.2$</td>\n",
       "      <td>$71.66 ± 0.5$</td>\n",
       "      <td>$70.83 ± 0.7$</td>\n",
       "      <td>$72.36 ± 0.3$</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">token_dev</th>\n",
       "      <th>multitok</th>\n",
       "      <td>$76.53 ± 1.0$</td>\n",
       "      <td>$78.76 ± 1.3$</td>\n",
       "      <td>$80.93 ± 0.7$</td>\n",
       "      <td>$67.47 ± 1.0$</td>\n",
       "      <td>$70.09 ± 1.1$</td>\n",
       "      <td>$72.89 ± 1.1$</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>token</th>\n",
       "      <td>$79.79 ± 0.8$</td>\n",
       "      <td>$81.11 ± 0.4$</td>\n",
       "      <td>$81.96 ± 0.8$</td>\n",
       "      <td>$66.43 ± 1.4$</td>\n",
       "      <td>$70.91 ± 0.9$</td>\n",
       "      <td>$72.97 ± 0.9$</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "morpheme_count                         1                                \\\n",
       "embed_type                           glv             ft         ft_oov   \n",
       "gold_name        unit                                                    \n",
       "morph_dev_gold   morph     $80.77 ± 0.8$  $82.33 ± 0.7$  $83.55 ± 1.2$   \n",
       "morph_dev_pruned morph     $79.32 ± 0.9$  $81.52 ± 0.5$  $82.46 ± 1.2$   \n",
       "token_dev        multitok  $76.53 ± 1.0$  $78.76 ± 1.3$  $80.93 ± 0.7$   \n",
       "                 token     $79.79 ± 0.8$  $81.11 ± 0.4$  $81.96 ± 0.8$   \n",
       "\n",
       "morpheme_count                         2                                \n",
       "embed_type                           glv             ft         ft_oov  \n",
       "gold_name        unit                                                   \n",
       "morph_dev_gold   morph     $74.37 ± 0.7$  $74.53 ± 0.6$  $75.43 ± 0.5$  \n",
       "morph_dev_pruned morph     $71.66 ± 0.5$  $70.83 ± 0.7$  $72.36 ± 0.3$  \n",
       "token_dev        multitok  $67.47 ± 1.0$  $70.09 ± 1.1$  $72.89 ± 1.1$  \n",
       "                 token     $66.43 ± 1.4$  $70.91 ± 0.9$  $72.97 ± 0.9$  "
      ]
     },
     "execution_count": 317,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x = (gsc[((gsc.gold_name.isin(['morph_dev_gold','morph_dev_pruned']))\n",
    "      & (gsc.arch=='char_cnn')\n",
    "      & (gsc.w_embed!='no_word')\n",
    "      & (gsc.cm=='Match')\n",
    "\n",
    "     )\n",
    "    |\n",
    "     (\n",
    "        (gsc.gold_name.isin(['token_dev']))\n",
    "      & (gsc.arch=='char_lstm')\n",
    "      & (gsc.w_embed!='no_word')\n",
    "      & (gsc.cm=='Match')\n",
    "     )]\n",
    " .groupby(['gold_name', 'unit', 'morpheme_count', 'embed_type'])\n",
    " .f.agg(['mean', 'std']).mul(100).round(2)\n",
    "         .assign(mean = lambda x: '$'+x['mean'].apply('{:,.2f}'.format).astype(str)+' ± '+ x['std'].round(1).astype(str)+'$')[['mean']]\n",
    "         .unstack([-2,-1]))\n",
    "x.columns = x.columns.droplevel(0)\n",
    "x[[(1, 'glv'), (1, 'ft'), (1, 'ft_oov'), \n",
    "   (2, 'glv'), (2, 'ft'), (2, 'ft_oov'),  ]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 318,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\\begin{tabular}{llllllll}\n",
      "\\toprule\n",
      "          & \\textbf{morpheme\\_count} & \\multicolumn{3}{l}{1} & \\multicolumn{3}{l}{2} \\\\\n",
      "          & \\textbf{embed\\_type} &            glv &             ft &         ft\\_oov &            glv &             ft &         ft\\_oov \\\\\n",
      "\\textbf{gold\\_name} & \\textbf{unit} &                &                &                &                &                &                \\\\\n",
      "\\midrule\n",
      "\\textbf{morph\\_dev\\_gold} & \\textbf{morph} &  $80.77 \\pm 0.8$ &  $82.33 \\pm 0.7$ &  $83.55 \\pm 1.2$ &  $74.37 \\pm 0.7$ &  $74.53 \\pm 0.6$ &  $75.43 \\pm 0.5$ \\\\\n",
      "\\textbf{morph\\_dev\\_pruned} & \\textbf{morph} &  $79.32 \\pm 0.9$ &  $81.52 \\pm 0.5$ &  $82.46 \\pm 1.2$ &  $71.66 \\pm 0.5$ &  $70.83 \\pm 0.7$ &  $72.36 \\pm 0.3$ \\\\\n",
      "\\textbf{token\\_dev} & \\textbf{multitok} &  $76.53 \\pm 1.0$ &  $78.76 \\pm 1.3$ &  $80.93 \\pm 0.7$ &  $67.47 \\pm 1.0$ &  $70.09 \\pm 1.1$ &  $72.89 \\pm 1.1$ \\\\\n",
      "          & \\textbf{token} &  $79.79 \\pm 0.8$ &  $81.11 \\pm 0.4$ &  $81.96 \\pm 0.8$ &  $66.43 \\pm 1.4$ &  $70.91 \\pm 0.9$ &  $72.97 \\pm 0.9$ \\\\\n",
      "\\bottomrule\n",
      "\\end{tabular}\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print (x[[(1, 'glv'), (1, 'ft'), (1, 'ft_oov'), \n",
    "   (2, 'glv'), (2, 'ft'), (2, 'ft_oov'), \n",
    "]]\n",
    "       .to_latex(bold_rows=True).replace('±', '\\pm').replace('\\$', '$'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 275,
   "metadata": {},
   "outputs": [],
   "source": [
    "from scipy.stats import ttest_ind"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 283,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Ttest_indResult(statistic=2.5674395676052857, pvalue=0.01937938613779738)"
      ]
     },
     "execution_count": 283,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tt = (gsc[(gsc.unk_type=='Comp') & ( ((gsc.gold_name=='morph_dev_pruned')\n",
    "      & (gsc.arch=='char_cnn')\n",
    "      & (gsc.w_embed=='ft_oov_yap')\n",
    "\n",
    "     )\n",
    "    |\n",
    "     (\n",
    "        (gsc.gold_name.isin(['token_dev']))\n",
    "      & (gsc.arch=='char_lstm')\n",
    "      & (gsc.w_embed=='ft_oov_tok')\n",
    "\n",
    "     ))]\n",
    ")\n",
    "\n",
    "ttest_ind(tt[tt.unit=='morph'].f, tt[tt.unit=='token'].f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "gsc = group_scores(scores2, groupers=[ignore, max, ignore], ment_grouper=ignore)\n",
    "g = (gsc[gsc.gold_name.str.contains('morph_dev')].fillna('x')\n",
    " .groupby(['gold_name', 'unit', 'arch', 'w_embed', 'morpheme_count'], sort=False)\n",
    " [['true', 'pred', 'correct']].mean().unstack()).sort_index().round(1)\n",
    "g.reindex(sorted(g.columns), axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "g = (gsc[gsc.gold_name.str.contains('token_dev')].fillna('x')\n",
    " .groupby(['gold_name', 'unit', 'arch', 'w_embed', 'morpheme_count'], sort=False)\n",
    " .f.mean().sort_index().unstack())\n",
    "g.reindex(sorted(g.columns), axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Mention length"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 161,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>ment_len</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>gold_name</th>\n",
       "      <th>unit</th>\n",
       "      <th>arch</th>\n",
       "      <th>w_embed</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"21\" valign=\"top\">morph_dev_gold</th>\n",
       "      <th rowspan=\"21\" valign=\"top\">morph</th>\n",
       "      <th rowspan=\"7\" valign=\"top\">char_cnn</th>\n",
       "      <th>ft_oov_tok</th>\n",
       "      <td>84.66</td>\n",
       "      <td>82.72</td>\n",
       "      <td>64.99</td>\n",
       "      <td>63.72</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_oov_yap</th>\n",
       "      <td>84.18</td>\n",
       "      <td>83.90</td>\n",
       "      <td>65.69</td>\n",
       "      <td>64.77</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_tok</th>\n",
       "      <td>83.72</td>\n",
       "      <td>82.87</td>\n",
       "      <td>64.78</td>\n",
       "      <td>64.67</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_yap</th>\n",
       "      <td>82.41</td>\n",
       "      <td>82.99</td>\n",
       "      <td>66.12</td>\n",
       "      <td>64.93</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_tok</th>\n",
       "      <td>82.38</td>\n",
       "      <td>81.61</td>\n",
       "      <td>65.77</td>\n",
       "      <td>60.04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_yap</th>\n",
       "      <td>81.95</td>\n",
       "      <td>81.56</td>\n",
       "      <td>66.25</td>\n",
       "      <td>61.99</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>no_word</th>\n",
       "      <td>62.13</td>\n",
       "      <td>69.09</td>\n",
       "      <td>55.15</td>\n",
       "      <td>56.93</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"7\" valign=\"top\">char_lstm</th>\n",
       "      <th>ft_oov_tok</th>\n",
       "      <td>83.95</td>\n",
       "      <td>82.32</td>\n",
       "      <td>64.62</td>\n",
       "      <td>64.76</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_oov_yap</th>\n",
       "      <td>84.74</td>\n",
       "      <td>82.63</td>\n",
       "      <td>64.58</td>\n",
       "      <td>63.43</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_tok</th>\n",
       "      <td>83.17</td>\n",
       "      <td>82.02</td>\n",
       "      <td>64.20</td>\n",
       "      <td>64.88</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_yap</th>\n",
       "      <td>82.63</td>\n",
       "      <td>82.84</td>\n",
       "      <td>64.09</td>\n",
       "      <td>64.25</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_tok</th>\n",
       "      <td>82.55</td>\n",
       "      <td>80.91</td>\n",
       "      <td>64.28</td>\n",
       "      <td>62.13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_yap</th>\n",
       "      <td>82.44</td>\n",
       "      <td>81.25</td>\n",
       "      <td>69.12</td>\n",
       "      <td>63.91</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>no_word</th>\n",
       "      <td>62.39</td>\n",
       "      <td>69.21</td>\n",
       "      <td>55.27</td>\n",
       "      <td>54.18</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"7\" valign=\"top\">no_char</th>\n",
       "      <th>ft_oov_tok</th>\n",
       "      <td>83.99</td>\n",
       "      <td>82.61</td>\n",
       "      <td>65.75</td>\n",
       "      <td>65.72</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_oov_yap</th>\n",
       "      <td>83.98</td>\n",
       "      <td>83.08</td>\n",
       "      <td>66.47</td>\n",
       "      <td>63.81</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_tok</th>\n",
       "      <td>81.54</td>\n",
       "      <td>82.99</td>\n",
       "      <td>65.52</td>\n",
       "      <td>65.35</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_yap</th>\n",
       "      <td>80.29</td>\n",
       "      <td>83.09</td>\n",
       "      <td>65.90</td>\n",
       "      <td>64.88</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_tok</th>\n",
       "      <td>79.67</td>\n",
       "      <td>81.10</td>\n",
       "      <td>65.48</td>\n",
       "      <td>59.91</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_yap</th>\n",
       "      <td>78.78</td>\n",
       "      <td>80.99</td>\n",
       "      <td>66.68</td>\n",
       "      <td>62.85</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>no_word</th>\n",
       "      <td>59.99</td>\n",
       "      <td>62.95</td>\n",
       "      <td>56.51</td>\n",
       "      <td>53.74</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"21\" valign=\"top\">morph_dev_pruned</th>\n",
       "      <th rowspan=\"21\" valign=\"top\">morph</th>\n",
       "      <th rowspan=\"7\" valign=\"top\">char_cnn</th>\n",
       "      <th>ft_oov_tok</th>\n",
       "      <td>82.97</td>\n",
       "      <td>79.97</td>\n",
       "      <td>61.08</td>\n",
       "      <td>62.70</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_oov_yap</th>\n",
       "      <td>83.00</td>\n",
       "      <td>81.53</td>\n",
       "      <td>61.16</td>\n",
       "      <td>63.62</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_tok</th>\n",
       "      <td>80.27</td>\n",
       "      <td>79.32</td>\n",
       "      <td>61.52</td>\n",
       "      <td>63.76</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_yap</th>\n",
       "      <td>80.97</td>\n",
       "      <td>80.07</td>\n",
       "      <td>61.66</td>\n",
       "      <td>64.78</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_tok</th>\n",
       "      <td>80.13</td>\n",
       "      <td>78.05</td>\n",
       "      <td>61.11</td>\n",
       "      <td>59.07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_yap</th>\n",
       "      <td>80.76</td>\n",
       "      <td>77.74</td>\n",
       "      <td>63.11</td>\n",
       "      <td>62.55</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>no_word</th>\n",
       "      <td>57.92</td>\n",
       "      <td>65.81</td>\n",
       "      <td>49.87</td>\n",
       "      <td>53.73</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"7\" valign=\"top\">char_lstm</th>\n",
       "      <th>ft_oov_tok</th>\n",
       "      <td>81.40</td>\n",
       "      <td>79.65</td>\n",
       "      <td>61.08</td>\n",
       "      <td>63.93</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_oov_yap</th>\n",
       "      <td>83.29</td>\n",
       "      <td>80.31</td>\n",
       "      <td>62.39</td>\n",
       "      <td>63.39</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_tok</th>\n",
       "      <td>80.14</td>\n",
       "      <td>77.48</td>\n",
       "      <td>61.00</td>\n",
       "      <td>64.15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_yap</th>\n",
       "      <td>80.91</td>\n",
       "      <td>80.19</td>\n",
       "      <td>60.11</td>\n",
       "      <td>64.47</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_tok</th>\n",
       "      <td>79.66</td>\n",
       "      <td>77.62</td>\n",
       "      <td>60.09</td>\n",
       "      <td>60.45</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_yap</th>\n",
       "      <td>80.74</td>\n",
       "      <td>77.46</td>\n",
       "      <td>66.03</td>\n",
       "      <td>63.99</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>no_word</th>\n",
       "      <td>58.86</td>\n",
       "      <td>65.55</td>\n",
       "      <td>49.14</td>\n",
       "      <td>51.50</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"7\" valign=\"top\">no_char</th>\n",
       "      <th>ft_oov_tok</th>\n",
       "      <td>81.28</td>\n",
       "      <td>80.56</td>\n",
       "      <td>64.50</td>\n",
       "      <td>64.18</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_oov_yap</th>\n",
       "      <td>82.02</td>\n",
       "      <td>80.90</td>\n",
       "      <td>57.06</td>\n",
       "      <td>61.93</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_tok</th>\n",
       "      <td>77.65</td>\n",
       "      <td>79.64</td>\n",
       "      <td>62.82</td>\n",
       "      <td>65.34</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_yap</th>\n",
       "      <td>79.18</td>\n",
       "      <td>81.55</td>\n",
       "      <td>60.42</td>\n",
       "      <td>63.54</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_tok</th>\n",
       "      <td>77.44</td>\n",
       "      <td>77.78</td>\n",
       "      <td>62.19</td>\n",
       "      <td>59.14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_yap</th>\n",
       "      <td>76.76</td>\n",
       "      <td>77.18</td>\n",
       "      <td>63.59</td>\n",
       "      <td>62.12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>no_word</th>\n",
       "      <td>55.19</td>\n",
       "      <td>59.18</td>\n",
       "      <td>49.63</td>\n",
       "      <td>51.01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"21\" valign=\"top\">morph_dev_yap</th>\n",
       "      <th rowspan=\"21\" valign=\"top\">morph</th>\n",
       "      <th rowspan=\"7\" valign=\"top\">char_cnn</th>\n",
       "      <th>ft_oov_tok</th>\n",
       "      <td>75.45</td>\n",
       "      <td>74.62</td>\n",
       "      <td>60.89</td>\n",
       "      <td>63.04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_oov_yap</th>\n",
       "      <td>75.55</td>\n",
       "      <td>75.19</td>\n",
       "      <td>60.85</td>\n",
       "      <td>63.39</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_tok</th>\n",
       "      <td>74.27</td>\n",
       "      <td>74.42</td>\n",
       "      <td>58.67</td>\n",
       "      <td>63.55</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_yap</th>\n",
       "      <td>73.44</td>\n",
       "      <td>74.09</td>\n",
       "      <td>59.32</td>\n",
       "      <td>63.46</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_tok</th>\n",
       "      <td>73.59</td>\n",
       "      <td>72.30</td>\n",
       "      <td>59.07</td>\n",
       "      <td>59.29</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_yap</th>\n",
       "      <td>73.28</td>\n",
       "      <td>71.91</td>\n",
       "      <td>58.53</td>\n",
       "      <td>60.48</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>no_word</th>\n",
       "      <td>54.73</td>\n",
       "      <td>62.03</td>\n",
       "      <td>49.01</td>\n",
       "      <td>56.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"7\" valign=\"top\">char_lstm</th>\n",
       "      <th>ft_oov_tok</th>\n",
       "      <td>74.79</td>\n",
       "      <td>73.74</td>\n",
       "      <td>59.31</td>\n",
       "      <td>63.67</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_oov_yap</th>\n",
       "      <td>75.91</td>\n",
       "      <td>74.05</td>\n",
       "      <td>59.81</td>\n",
       "      <td>62.69</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_tok</th>\n",
       "      <td>73.78</td>\n",
       "      <td>73.15</td>\n",
       "      <td>57.66</td>\n",
       "      <td>63.59</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_yap</th>\n",
       "      <td>73.05</td>\n",
       "      <td>73.69</td>\n",
       "      <td>57.28</td>\n",
       "      <td>62.93</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_tok</th>\n",
       "      <td>73.36</td>\n",
       "      <td>71.62</td>\n",
       "      <td>57.55</td>\n",
       "      <td>61.63</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_yap</th>\n",
       "      <td>73.30</td>\n",
       "      <td>71.71</td>\n",
       "      <td>61.10</td>\n",
       "      <td>63.36</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>no_word</th>\n",
       "      <td>55.02</td>\n",
       "      <td>62.93</td>\n",
       "      <td>50.82</td>\n",
       "      <td>53.50</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"7\" valign=\"top\">no_char</th>\n",
       "      <th>ft_oov_tok</th>\n",
       "      <td>74.48</td>\n",
       "      <td>74.02</td>\n",
       "      <td>61.58</td>\n",
       "      <td>64.49</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_oov_yap</th>\n",
       "      <td>75.64</td>\n",
       "      <td>74.51</td>\n",
       "      <td>61.66</td>\n",
       "      <td>63.25</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_tok</th>\n",
       "      <td>72.68</td>\n",
       "      <td>74.10</td>\n",
       "      <td>59.17</td>\n",
       "      <td>64.53</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_yap</th>\n",
       "      <td>71.57</td>\n",
       "      <td>74.60</td>\n",
       "      <td>59.38</td>\n",
       "      <td>63.91</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_tok</th>\n",
       "      <td>70.93</td>\n",
       "      <td>72.39</td>\n",
       "      <td>60.18</td>\n",
       "      <td>59.64</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_yap</th>\n",
       "      <td>69.95</td>\n",
       "      <td>72.03</td>\n",
       "      <td>60.70</td>\n",
       "      <td>62.01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>no_word</th>\n",
       "      <td>52.36</td>\n",
       "      <td>56.50</td>\n",
       "      <td>52.71</td>\n",
       "      <td>54.02</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"42\" valign=\"top\">token_dev</th>\n",
       "      <th rowspan=\"21\" valign=\"top\">multitok</th>\n",
       "      <th rowspan=\"7\" valign=\"top\">char_cnn</th>\n",
       "      <th>ft_oov_tok</th>\n",
       "      <td>77.80</td>\n",
       "      <td>81.14</td>\n",
       "      <td>61.69</td>\n",
       "      <td>42.78</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_oov_yap</th>\n",
       "      <td>76.27</td>\n",
       "      <td>77.59</td>\n",
       "      <td>61.62</td>\n",
       "      <td>47.33</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_tok</th>\n",
       "      <td>74.46</td>\n",
       "      <td>80.60</td>\n",
       "      <td>60.51</td>\n",
       "      <td>42.99</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_yap</th>\n",
       "      <td>71.22</td>\n",
       "      <td>75.82</td>\n",
       "      <td>55.65</td>\n",
       "      <td>42.15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_tok</th>\n",
       "      <td>72.69</td>\n",
       "      <td>77.43</td>\n",
       "      <td>59.81</td>\n",
       "      <td>39.86</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_yap</th>\n",
       "      <td>70.50</td>\n",
       "      <td>73.43</td>\n",
       "      <td>56.49</td>\n",
       "      <td>41.33</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>no_word</th>\n",
       "      <td>53.88</td>\n",
       "      <td>60.96</td>\n",
       "      <td>43.21</td>\n",
       "      <td>27.14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"7\" valign=\"top\">char_lstm</th>\n",
       "      <th>ft_oov_tok</th>\n",
       "      <td>78.95</td>\n",
       "      <td>81.09</td>\n",
       "      <td>61.71</td>\n",
       "      <td>43.99</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_oov_yap</th>\n",
       "      <td>76.21</td>\n",
       "      <td>78.68</td>\n",
       "      <td>61.70</td>\n",
       "      <td>52.33</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_tok</th>\n",
       "      <td>74.61</td>\n",
       "      <td>80.37</td>\n",
       "      <td>62.52</td>\n",
       "      <td>48.45</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_yap</th>\n",
       "      <td>72.37</td>\n",
       "      <td>75.65</td>\n",
       "      <td>55.13</td>\n",
       "      <td>44.29</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_tok</th>\n",
       "      <td>73.31</td>\n",
       "      <td>76.93</td>\n",
       "      <td>58.87</td>\n",
       "      <td>41.12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_yap</th>\n",
       "      <td>69.22</td>\n",
       "      <td>73.30</td>\n",
       "      <td>55.15</td>\n",
       "      <td>40.52</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>no_word</th>\n",
       "      <td>54.50</td>\n",
       "      <td>60.94</td>\n",
       "      <td>45.36</td>\n",
       "      <td>26.78</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"7\" valign=\"top\">no_char</th>\n",
       "      <th>ft_oov_tok</th>\n",
       "      <td>79.91</td>\n",
       "      <td>81.42</td>\n",
       "      <td>60.15</td>\n",
       "      <td>46.34</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_oov_yap</th>\n",
       "      <td>74.97</td>\n",
       "      <td>77.76</td>\n",
       "      <td>56.13</td>\n",
       "      <td>47.37</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_tok</th>\n",
       "      <td>75.00</td>\n",
       "      <td>80.01</td>\n",
       "      <td>57.27</td>\n",
       "      <td>45.08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_yap</th>\n",
       "      <td>69.57</td>\n",
       "      <td>72.51</td>\n",
       "      <td>53.79</td>\n",
       "      <td>25.72</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_tok</th>\n",
       "      <td>70.26</td>\n",
       "      <td>76.03</td>\n",
       "      <td>56.82</td>\n",
       "      <td>33.86</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_yap</th>\n",
       "      <td>66.03</td>\n",
       "      <td>70.25</td>\n",
       "      <td>50.25</td>\n",
       "      <td>23.18</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>no_word</th>\n",
       "      <td>48.53</td>\n",
       "      <td>52.34</td>\n",
       "      <td>40.87</td>\n",
       "      <td>14.63</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"21\" valign=\"top\">token</th>\n",
       "      <th rowspan=\"7\" valign=\"top\">char_cnn</th>\n",
       "      <th>ft_oov_tok</th>\n",
       "      <td>79.75</td>\n",
       "      <td>82.34</td>\n",
       "      <td>62.61</td>\n",
       "      <td>43.86</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_oov_yap</th>\n",
       "      <td>77.09</td>\n",
       "      <td>78.18</td>\n",
       "      <td>66.32</td>\n",
       "      <td>51.01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_tok</th>\n",
       "      <td>76.12</td>\n",
       "      <td>80.75</td>\n",
       "      <td>60.65</td>\n",
       "      <td>48.85</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_yap</th>\n",
       "      <td>70.77</td>\n",
       "      <td>75.31</td>\n",
       "      <td>55.96</td>\n",
       "      <td>40.71</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_tok</th>\n",
       "      <td>75.37</td>\n",
       "      <td>77.14</td>\n",
       "      <td>62.03</td>\n",
       "      <td>43.20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_yap</th>\n",
       "      <td>69.60</td>\n",
       "      <td>73.68</td>\n",
       "      <td>54.36</td>\n",
       "      <td>27.35</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>no_word</th>\n",
       "      <td>56.42</td>\n",
       "      <td>60.72</td>\n",
       "      <td>43.96</td>\n",
       "      <td>36.64</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"7\" valign=\"top\">char_lstm</th>\n",
       "      <th>ft_oov_tok</th>\n",
       "      <td>79.35</td>\n",
       "      <td>82.12</td>\n",
       "      <td>62.22</td>\n",
       "      <td>47.19</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_oov_yap</th>\n",
       "      <td>77.02</td>\n",
       "      <td>78.01</td>\n",
       "      <td>63.90</td>\n",
       "      <td>51.15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_tok</th>\n",
       "      <td>77.58</td>\n",
       "      <td>81.18</td>\n",
       "      <td>60.46</td>\n",
       "      <td>50.69</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_yap</th>\n",
       "      <td>71.90</td>\n",
       "      <td>74.26</td>\n",
       "      <td>55.40</td>\n",
       "      <td>44.83</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_tok</th>\n",
       "      <td>75.69</td>\n",
       "      <td>77.29</td>\n",
       "      <td>60.10</td>\n",
       "      <td>42.36</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_yap</th>\n",
       "      <td>70.24</td>\n",
       "      <td>73.91</td>\n",
       "      <td>55.37</td>\n",
       "      <td>23.27</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>no_word</th>\n",
       "      <td>56.23</td>\n",
       "      <td>60.34</td>\n",
       "      <td>45.69</td>\n",
       "      <td>38.62</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"7\" valign=\"top\">no_char</th>\n",
       "      <th>ft_oov_tok</th>\n",
       "      <td>78.73</td>\n",
       "      <td>81.26</td>\n",
       "      <td>61.89</td>\n",
       "      <td>42.71</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_oov_yap</th>\n",
       "      <td>75.62</td>\n",
       "      <td>77.64</td>\n",
       "      <td>64.02</td>\n",
       "      <td>50.89</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_tok</th>\n",
       "      <td>74.43</td>\n",
       "      <td>80.62</td>\n",
       "      <td>58.81</td>\n",
       "      <td>47.64</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_yap</th>\n",
       "      <td>70.21</td>\n",
       "      <td>73.46</td>\n",
       "      <td>54.47</td>\n",
       "      <td>28.88</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_tok</th>\n",
       "      <td>70.94</td>\n",
       "      <td>76.38</td>\n",
       "      <td>59.52</td>\n",
       "      <td>37.26</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_yap</th>\n",
       "      <td>66.06</td>\n",
       "      <td>72.36</td>\n",
       "      <td>53.18</td>\n",
       "      <td>20.17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>no_word</th>\n",
       "      <td>53.92</td>\n",
       "      <td>55.47</td>\n",
       "      <td>41.06</td>\n",
       "      <td>31.84</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "ment_len                                            1      2      3      4\n",
       "gold_name        unit     arch      w_embed                               \n",
       "morph_dev_gold   morph    char_cnn  ft_oov_tok  84.66  82.72  64.99  63.72\n",
       "                                    ft_oov_yap  84.18  83.90  65.69  64.77\n",
       "                                    ft_tok      83.72  82.87  64.78  64.67\n",
       "                                    ft_yap      82.41  82.99  66.12  64.93\n",
       "                                    glv_tok     82.38  81.61  65.77  60.04\n",
       "                                    glv_yap     81.95  81.56  66.25  61.99\n",
       "                                    no_word     62.13  69.09  55.15  56.93\n",
       "                          char_lstm ft_oov_tok  83.95  82.32  64.62  64.76\n",
       "                                    ft_oov_yap  84.74  82.63  64.58  63.43\n",
       "                                    ft_tok      83.17  82.02  64.20  64.88\n",
       "                                    ft_yap      82.63  82.84  64.09  64.25\n",
       "                                    glv_tok     82.55  80.91  64.28  62.13\n",
       "                                    glv_yap     82.44  81.25  69.12  63.91\n",
       "                                    no_word     62.39  69.21  55.27  54.18\n",
       "                          no_char   ft_oov_tok  83.99  82.61  65.75  65.72\n",
       "                                    ft_oov_yap  83.98  83.08  66.47  63.81\n",
       "                                    ft_tok      81.54  82.99  65.52  65.35\n",
       "                                    ft_yap      80.29  83.09  65.90  64.88\n",
       "                                    glv_tok     79.67  81.10  65.48  59.91\n",
       "                                    glv_yap     78.78  80.99  66.68  62.85\n",
       "                                    no_word     59.99  62.95  56.51  53.74\n",
       "morph_dev_pruned morph    char_cnn  ft_oov_tok  82.97  79.97  61.08  62.70\n",
       "                                    ft_oov_yap  83.00  81.53  61.16  63.62\n",
       "                                    ft_tok      80.27  79.32  61.52  63.76\n",
       "                                    ft_yap      80.97  80.07  61.66  64.78\n",
       "                                    glv_tok     80.13  78.05  61.11  59.07\n",
       "                                    glv_yap     80.76  77.74  63.11  62.55\n",
       "                                    no_word     57.92  65.81  49.87  53.73\n",
       "                          char_lstm ft_oov_tok  81.40  79.65  61.08  63.93\n",
       "                                    ft_oov_yap  83.29  80.31  62.39  63.39\n",
       "                                    ft_tok      80.14  77.48  61.00  64.15\n",
       "                                    ft_yap      80.91  80.19  60.11  64.47\n",
       "                                    glv_tok     79.66  77.62  60.09  60.45\n",
       "                                    glv_yap     80.74  77.46  66.03  63.99\n",
       "                                    no_word     58.86  65.55  49.14  51.50\n",
       "                          no_char   ft_oov_tok  81.28  80.56  64.50  64.18\n",
       "                                    ft_oov_yap  82.02  80.90  57.06  61.93\n",
       "                                    ft_tok      77.65  79.64  62.82  65.34\n",
       "                                    ft_yap      79.18  81.55  60.42  63.54\n",
       "                                    glv_tok     77.44  77.78  62.19  59.14\n",
       "                                    glv_yap     76.76  77.18  63.59  62.12\n",
       "                                    no_word     55.19  59.18  49.63  51.01\n",
       "morph_dev_yap    morph    char_cnn  ft_oov_tok  75.45  74.62  60.89  63.04\n",
       "                                    ft_oov_yap  75.55  75.19  60.85  63.39\n",
       "                                    ft_tok      74.27  74.42  58.67  63.55\n",
       "                                    ft_yap      73.44  74.09  59.32  63.46\n",
       "                                    glv_tok     73.59  72.30  59.07  59.29\n",
       "                                    glv_yap     73.28  71.91  58.53  60.48\n",
       "                                    no_word     54.73  62.03  49.01  56.00\n",
       "                          char_lstm ft_oov_tok  74.79  73.74  59.31  63.67\n",
       "                                    ft_oov_yap  75.91  74.05  59.81  62.69\n",
       "                                    ft_tok      73.78  73.15  57.66  63.59\n",
       "                                    ft_yap      73.05  73.69  57.28  62.93\n",
       "                                    glv_tok     73.36  71.62  57.55  61.63\n",
       "                                    glv_yap     73.30  71.71  61.10  63.36\n",
       "                                    no_word     55.02  62.93  50.82  53.50\n",
       "                          no_char   ft_oov_tok  74.48  74.02  61.58  64.49\n",
       "                                    ft_oov_yap  75.64  74.51  61.66  63.25\n",
       "                                    ft_tok      72.68  74.10  59.17  64.53\n",
       "                                    ft_yap      71.57  74.60  59.38  63.91\n",
       "                                    glv_tok     70.93  72.39  60.18  59.64\n",
       "                                    glv_yap     69.95  72.03  60.70  62.01\n",
       "                                    no_word     52.36  56.50  52.71  54.02\n",
       "token_dev        multitok char_cnn  ft_oov_tok  77.80  81.14  61.69  42.78\n",
       "                                    ft_oov_yap  76.27  77.59  61.62  47.33\n",
       "                                    ft_tok      74.46  80.60  60.51  42.99\n",
       "                                    ft_yap      71.22  75.82  55.65  42.15\n",
       "                                    glv_tok     72.69  77.43  59.81  39.86\n",
       "                                    glv_yap     70.50  73.43  56.49  41.33\n",
       "                                    no_word     53.88  60.96  43.21  27.14\n",
       "                          char_lstm ft_oov_tok  78.95  81.09  61.71  43.99\n",
       "                                    ft_oov_yap  76.21  78.68  61.70  52.33\n",
       "                                    ft_tok      74.61  80.37  62.52  48.45\n",
       "                                    ft_yap      72.37  75.65  55.13  44.29\n",
       "                                    glv_tok     73.31  76.93  58.87  41.12\n",
       "                                    glv_yap     69.22  73.30  55.15  40.52\n",
       "                                    no_word     54.50  60.94  45.36  26.78\n",
       "                          no_char   ft_oov_tok  79.91  81.42  60.15  46.34\n",
       "                                    ft_oov_yap  74.97  77.76  56.13  47.37\n",
       "                                    ft_tok      75.00  80.01  57.27  45.08\n",
       "                                    ft_yap      69.57  72.51  53.79  25.72\n",
       "                                    glv_tok     70.26  76.03  56.82  33.86\n",
       "                                    glv_yap     66.03  70.25  50.25  23.18\n",
       "                                    no_word     48.53  52.34  40.87  14.63\n",
       "                 token    char_cnn  ft_oov_tok  79.75  82.34  62.61  43.86\n",
       "                                    ft_oov_yap  77.09  78.18  66.32  51.01\n",
       "                                    ft_tok      76.12  80.75  60.65  48.85\n",
       "                                    ft_yap      70.77  75.31  55.96  40.71\n",
       "                                    glv_tok     75.37  77.14  62.03  43.20\n",
       "                                    glv_yap     69.60  73.68  54.36  27.35\n",
       "                                    no_word     56.42  60.72  43.96  36.64\n",
       "                          char_lstm ft_oov_tok  79.35  82.12  62.22  47.19\n",
       "                                    ft_oov_yap  77.02  78.01  63.90  51.15\n",
       "                                    ft_tok      77.58  81.18  60.46  50.69\n",
       "                                    ft_yap      71.90  74.26  55.40  44.83\n",
       "                                    glv_tok     75.69  77.29  60.10  42.36\n",
       "                                    glv_yap     70.24  73.91  55.37  23.27\n",
       "                                    no_word     56.23  60.34  45.69  38.62\n",
       "                          no_char   ft_oov_tok  78.73  81.26  61.89  42.71\n",
       "                                    ft_oov_yap  75.62  77.64  64.02  50.89\n",
       "                                    ft_tok      74.43  80.62  58.81  47.64\n",
       "                                    ft_yap      70.21  73.46  54.47  28.88\n",
       "                                    glv_tok     70.94  76.38  59.52  37.26\n",
       "                                    glv_yap     66.06  72.36  53.18  20.17\n",
       "                                    no_word     53.92  55.47  41.06  31.84"
      ]
     },
     "execution_count": 161,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gsc = group_scores(scores4, groupers=[ignore, ignore, ignore], ment_grouper=four_up)\n",
    "(gsc[gsc.gold_name.str.contains('dev')].fillna('x')\n",
    " .groupby(['gold_name', 'unit', 'arch', 'w_embed', 'ment_len'])\n",
    " .f.mean().mul(100).round(2).unstack())\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 582,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>ment_len</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>gold_name</th>\n",
       "      <th>unit</th>\n",
       "      <th>arch</th>\n",
       "      <th>w_embed</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"42\" valign=\"top\">token_dev</th>\n",
       "      <th rowspan=\"21\" valign=\"top\">multitok</th>\n",
       "      <th rowspan=\"7\" valign=\"top\">char_lstm</th>\n",
       "      <th>glv_yap</th>\n",
       "      <td>0.699029</td>\n",
       "      <td>0.733733</td>\n",
       "      <td>0.555272</td>\n",
       "      <td>0.413200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_oov_yap</th>\n",
       "      <td>0.761828</td>\n",
       "      <td>0.788465</td>\n",
       "      <td>0.620608</td>\n",
       "      <td>0.489524</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_yap</th>\n",
       "      <td>0.725362</td>\n",
       "      <td>0.757029</td>\n",
       "      <td>0.552000</td>\n",
       "      <td>0.450476</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_oov_tok</th>\n",
       "      <td>0.784379</td>\n",
       "      <td>0.810604</td>\n",
       "      <td>0.615246</td>\n",
       "      <td>0.464762</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>no_word</th>\n",
       "      <td>0.546456</td>\n",
       "      <td>0.608189</td>\n",
       "      <td>0.455459</td>\n",
       "      <td>0.292277</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_tok</th>\n",
       "      <td>0.726478</td>\n",
       "      <td>0.773129</td>\n",
       "      <td>0.586930</td>\n",
       "      <td>0.389474</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_tok</th>\n",
       "      <td>0.746806</td>\n",
       "      <td>0.804697</td>\n",
       "      <td>0.611426</td>\n",
       "      <td>0.473734</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"7\" valign=\"top\">no_char</th>\n",
       "      <th>glv_yap</th>\n",
       "      <td>0.657524</td>\n",
       "      <td>0.700506</td>\n",
       "      <td>0.498668</td>\n",
       "      <td>0.188712</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_oov_yap</th>\n",
       "      <td>0.745697</td>\n",
       "      <td>0.775270</td>\n",
       "      <td>0.562606</td>\n",
       "      <td>0.447049</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_yap</th>\n",
       "      <td>0.695767</td>\n",
       "      <td>0.726564</td>\n",
       "      <td>0.536864</td>\n",
       "      <td>0.255766</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_oov_tok</th>\n",
       "      <td>0.803346</td>\n",
       "      <td>0.814032</td>\n",
       "      <td>0.599927</td>\n",
       "      <td>0.486409</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>no_word</th>\n",
       "      <td>0.486396</td>\n",
       "      <td>0.525734</td>\n",
       "      <td>0.405665</td>\n",
       "      <td>0.151654</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_tok</th>\n",
       "      <td>0.701759</td>\n",
       "      <td>0.761577</td>\n",
       "      <td>0.568133</td>\n",
       "      <td>0.345853</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_tok</th>\n",
       "      <td>0.748602</td>\n",
       "      <td>0.798457</td>\n",
       "      <td>0.573925</td>\n",
       "      <td>0.451034</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"7\" valign=\"top\">char_cnn</th>\n",
       "      <th>glv_yap</th>\n",
       "      <td>0.698517</td>\n",
       "      <td>0.735627</td>\n",
       "      <td>0.558933</td>\n",
       "      <td>0.411429</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_oov_yap</th>\n",
       "      <td>0.763519</td>\n",
       "      <td>0.776049</td>\n",
       "      <td>0.615391</td>\n",
       "      <td>0.455758</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_yap</th>\n",
       "      <td>0.712203</td>\n",
       "      <td>0.758217</td>\n",
       "      <td>0.556519</td>\n",
       "      <td>0.421536</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_oov_tok</th>\n",
       "      <td>0.774597</td>\n",
       "      <td>0.808944</td>\n",
       "      <td>0.626337</td>\n",
       "      <td>0.483377</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>no_word</th>\n",
       "      <td>0.541179</td>\n",
       "      <td>0.608129</td>\n",
       "      <td>0.438947</td>\n",
       "      <td>0.291395</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_tok</th>\n",
       "      <td>0.729265</td>\n",
       "      <td>0.778832</td>\n",
       "      <td>0.601311</td>\n",
       "      <td>0.415639</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_tok</th>\n",
       "      <td>0.741900</td>\n",
       "      <td>0.805653</td>\n",
       "      <td>0.602329</td>\n",
       "      <td>0.424211</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"21\" valign=\"top\">token</th>\n",
       "      <th rowspan=\"7\" valign=\"top\">char_lstm</th>\n",
       "      <th>glv_yap</th>\n",
       "      <td>0.700136</td>\n",
       "      <td>0.739657</td>\n",
       "      <td>0.556682</td>\n",
       "      <td>0.272348</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_oov_yap</th>\n",
       "      <td>0.771904</td>\n",
       "      <td>0.781070</td>\n",
       "      <td>0.631965</td>\n",
       "      <td>0.514340</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_yap</th>\n",
       "      <td>0.717893</td>\n",
       "      <td>0.744241</td>\n",
       "      <td>0.556784</td>\n",
       "      <td>0.445184</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_oov_tok</th>\n",
       "      <td>0.793525</td>\n",
       "      <td>0.821159</td>\n",
       "      <td>0.622206</td>\n",
       "      <td>0.471861</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>no_word</th>\n",
       "      <td>0.561934</td>\n",
       "      <td>0.600432</td>\n",
       "      <td>0.448080</td>\n",
       "      <td>0.389278</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_tok</th>\n",
       "      <td>0.756017</td>\n",
       "      <td>0.772499</td>\n",
       "      <td>0.602589</td>\n",
       "      <td>0.420587</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_tok</th>\n",
       "      <td>0.776515</td>\n",
       "      <td>0.813706</td>\n",
       "      <td>0.600388</td>\n",
       "      <td>0.503092</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"7\" valign=\"top\">no_char</th>\n",
       "      <th>glv_yap</th>\n",
       "      <td>0.660745</td>\n",
       "      <td>0.723062</td>\n",
       "      <td>0.538836</td>\n",
       "      <td>0.175172</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_oov_yap</th>\n",
       "      <td>0.756193</td>\n",
       "      <td>0.776365</td>\n",
       "      <td>0.640224</td>\n",
       "      <td>0.508903</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_yap</th>\n",
       "      <td>0.702124</td>\n",
       "      <td>0.734639</td>\n",
       "      <td>0.544667</td>\n",
       "      <td>0.288822</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_oov_tok</th>\n",
       "      <td>0.789761</td>\n",
       "      <td>0.814234</td>\n",
       "      <td>0.612996</td>\n",
       "      <td>0.407125</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>no_word</th>\n",
       "      <td>0.538321</td>\n",
       "      <td>0.550956</td>\n",
       "      <td>0.410934</td>\n",
       "      <td>0.336959</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_tok</th>\n",
       "      <td>0.708212</td>\n",
       "      <td>0.762532</td>\n",
       "      <td>0.614444</td>\n",
       "      <td>0.353383</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_tok</th>\n",
       "      <td>0.744900</td>\n",
       "      <td>0.806901</td>\n",
       "      <td>0.591698</td>\n",
       "      <td>0.477528</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"7\" valign=\"top\">char_cnn</th>\n",
       "      <th>glv_yap</th>\n",
       "      <td>0.696574</td>\n",
       "      <td>0.737746</td>\n",
       "      <td>0.544838</td>\n",
       "      <td>0.277344</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_oov_yap</th>\n",
       "      <td>0.777077</td>\n",
       "      <td>0.777736</td>\n",
       "      <td>0.680300</td>\n",
       "      <td>0.504348</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_yap</th>\n",
       "      <td>0.705519</td>\n",
       "      <td>0.753632</td>\n",
       "      <td>0.552980</td>\n",
       "      <td>0.411905</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_oov_tok</th>\n",
       "      <td>0.798248</td>\n",
       "      <td>0.824216</td>\n",
       "      <td>0.625852</td>\n",
       "      <td>0.417538</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>no_word</th>\n",
       "      <td>0.566245</td>\n",
       "      <td>0.606834</td>\n",
       "      <td>0.434072</td>\n",
       "      <td>0.371962</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>glv_tok</th>\n",
       "      <td>0.751950</td>\n",
       "      <td>0.770538</td>\n",
       "      <td>0.619574</td>\n",
       "      <td>0.447890</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ft_tok</th>\n",
       "      <td>0.762446</td>\n",
       "      <td>0.807673</td>\n",
       "      <td>0.595774</td>\n",
       "      <td>0.504762</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "ment_len                                        1         2         3  \\\n",
       "gold_name unit     arch      w_embed                                    \n",
       "token_dev multitok char_lstm glv_yap     0.699029  0.733733  0.555272   \n",
       "                             ft_oov_yap  0.761828  0.788465  0.620608   \n",
       "                             ft_yap      0.725362  0.757029  0.552000   \n",
       "                             ft_oov_tok  0.784379  0.810604  0.615246   \n",
       "                             no_word     0.546456  0.608189  0.455459   \n",
       "                             glv_tok     0.726478  0.773129  0.586930   \n",
       "                             ft_tok      0.746806  0.804697  0.611426   \n",
       "                   no_char   glv_yap     0.657524  0.700506  0.498668   \n",
       "                             ft_oov_yap  0.745697  0.775270  0.562606   \n",
       "                             ft_yap      0.695767  0.726564  0.536864   \n",
       "                             ft_oov_tok  0.803346  0.814032  0.599927   \n",
       "                             no_word     0.486396  0.525734  0.405665   \n",
       "                             glv_tok     0.701759  0.761577  0.568133   \n",
       "                             ft_tok      0.748602  0.798457  0.573925   \n",
       "                   char_cnn  glv_yap     0.698517  0.735627  0.558933   \n",
       "                             ft_oov_yap  0.763519  0.776049  0.615391   \n",
       "                             ft_yap      0.712203  0.758217  0.556519   \n",
       "                             ft_oov_tok  0.774597  0.808944  0.626337   \n",
       "                             no_word     0.541179  0.608129  0.438947   \n",
       "                             glv_tok     0.729265  0.778832  0.601311   \n",
       "                             ft_tok      0.741900  0.805653  0.602329   \n",
       "          token    char_lstm glv_yap     0.700136  0.739657  0.556682   \n",
       "                             ft_oov_yap  0.771904  0.781070  0.631965   \n",
       "                             ft_yap      0.717893  0.744241  0.556784   \n",
       "                             ft_oov_tok  0.793525  0.821159  0.622206   \n",
       "                             no_word     0.561934  0.600432  0.448080   \n",
       "                             glv_tok     0.756017  0.772499  0.602589   \n",
       "                             ft_tok      0.776515  0.813706  0.600388   \n",
       "                   no_char   glv_yap     0.660745  0.723062  0.538836   \n",
       "                             ft_oov_yap  0.756193  0.776365  0.640224   \n",
       "                             ft_yap      0.702124  0.734639  0.544667   \n",
       "                             ft_oov_tok  0.789761  0.814234  0.612996   \n",
       "                             no_word     0.538321  0.550956  0.410934   \n",
       "                             glv_tok     0.708212  0.762532  0.614444   \n",
       "                             ft_tok      0.744900  0.806901  0.591698   \n",
       "                   char_cnn  glv_yap     0.696574  0.737746  0.544838   \n",
       "                             ft_oov_yap  0.777077  0.777736  0.680300   \n",
       "                             ft_yap      0.705519  0.753632  0.552980   \n",
       "                             ft_oov_tok  0.798248  0.824216  0.625852   \n",
       "                             no_word     0.566245  0.606834  0.434072   \n",
       "                             glv_tok     0.751950  0.770538  0.619574   \n",
       "                             ft_tok      0.762446  0.807673  0.595774   \n",
       "\n",
       "ment_len                                        4  \n",
       "gold_name unit     arch      w_embed               \n",
       "token_dev multitok char_lstm glv_yap     0.413200  \n",
       "                             ft_oov_yap  0.489524  \n",
       "                             ft_yap      0.450476  \n",
       "                             ft_oov_tok  0.464762  \n",
       "                             no_word     0.292277  \n",
       "                             glv_tok     0.389474  \n",
       "                             ft_tok      0.473734  \n",
       "                   no_char   glv_yap     0.188712  \n",
       "                             ft_oov_yap  0.447049  \n",
       "                             ft_yap      0.255766  \n",
       "                             ft_oov_tok  0.486409  \n",
       "                             no_word     0.151654  \n",
       "                             glv_tok     0.345853  \n",
       "                             ft_tok      0.451034  \n",
       "                   char_cnn  glv_yap     0.411429  \n",
       "                             ft_oov_yap  0.455758  \n",
       "                             ft_yap      0.421536  \n",
       "                             ft_oov_tok  0.483377  \n",
       "                             no_word     0.291395  \n",
       "                             glv_tok     0.415639  \n",
       "                             ft_tok      0.424211  \n",
       "          token    char_lstm glv_yap     0.272348  \n",
       "                             ft_oov_yap  0.514340  \n",
       "                             ft_yap      0.445184  \n",
       "                             ft_oov_tok  0.471861  \n",
       "                             no_word     0.389278  \n",
       "                             glv_tok     0.420587  \n",
       "                             ft_tok      0.503092  \n",
       "                   no_char   glv_yap     0.175172  \n",
       "                             ft_oov_yap  0.508903  \n",
       "                             ft_yap      0.288822  \n",
       "                             ft_oov_tok  0.407125  \n",
       "                             no_word     0.336959  \n",
       "                             glv_tok     0.353383  \n",
       "                             ft_tok      0.477528  \n",
       "                   char_cnn  glv_yap     0.277344  \n",
       "                             ft_oov_yap  0.504348  \n",
       "                             ft_yap      0.411905  \n",
       "                             ft_oov_tok  0.417538  \n",
       "                             no_word     0.371962  \n",
       "                             glv_tok     0.447890  \n",
       "                             ft_tok      0.504762  "
      ]
     },
     "execution_count": 582,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "g = (gsc[gsc.gold_name.str.contains('token_dev')].fillna('x')\n",
    " .groupby(['gold_name', 'unit', 'arch', 'w_embed', 'ment_len'], sort=False)\n",
    " .f.mean().unstack())\n",
    "g.reindex(sorted(g.columns), axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pd.set_option(\"max_rows\", 100)\n",
    "gsc[gsc.gold_name.str.contains('morph_dev')].fillna('x').groupby(['gold_name', 'unit', 'arch', 'w_embed', 'unk_type', 'ooev'], sort=False).f.mean().unstack([-1, -2])"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "tf",
   "language": "python",
   "name": "tf"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}