NERNN.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# NERNN: Named Entity Recognition with Word Embeddings and Char RNNs\n",
    "- Use a combination of word embeddings and character embeddings + RNN to predict whether a word is a named entity"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 603,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The autoreload extension is already loaded. To reload it, use:\n",
      "  %reload_ext autoreload\n"
     ]
    }
   ],
   "source": [
    "# Imports\n",
    "import os\n",
    "import gzip\n",
    "import glob\n",
    "import re\n",
    "import json\n",
    "import h5py\n",
    "\n",
    "# Computational imports\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import tensorflow as tf\n",
    "tf.reset_default_graph()\n",
    "\n",
    "# Keras imports\n",
    "import keras\n",
    "import keras.backend as K\n",
    "from keras.preprocessing import sequence\n",
    "from keras.models import Sequential, Model\n",
    "from keras.layers import Layer, Input, Embedding, merge, Dense, GRU, TimeDistributed, Layer, Bidirectional, LSTM\n",
    "from model_saver import ModelCheckpointBatch\n",
    "\n",
    "# gensim imports\n",
    "import gensim\n",
    "from gensim.models import Word2Vec\n",
    "\n",
    "# plotting\n",
    "import matplotlib\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "matplotlib.rcParams['figure.figsize'] = (15, 8)\n",
    "plt.style.use('seaborn-whitegrid')\n",
    "\n",
    "%load_ext autoreload\n",
    "%autoreload"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# I. Define Model Architecture"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Model Vars"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 592,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "MODEL_SAVE_PATH = 'models/weights.{epoch:02d}-{batch:02d}.hdf5'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## a. Defining the Keras model architecture"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 593,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Going to need a custom layer for selecting the end of words in the character RNN\n",
    "class GatherLayer(Layer):\n",
    "    '''\n",
    "    Scans over the batch to gather specific indices along the time axis\n",
    "    '''\n",
    "    def __init__(self, **kwargs):\n",
    "        super(GatherLayer, self).__init__(**kwargs)\n",
    "    \n",
    "    def build(self, input_shape):\n",
    "        super(GatherLayer, self).build(input_shape)\n",
    "        \n",
    "    def compute_mask(self, x, mask=None):\n",
    "        '''\n",
    "        Compute the mask\n",
    "        '''\n",
    "        return K.cast(K.not_equal(x[1], -1), 'bool')\n",
    "    \n",
    "    def call(self, inputs, mask=None):\n",
    "        '''\n",
    "        First input is the rnn out (batch_size, max_word_steps, char_lstm_dim)\n",
    "        Second input is indicies to gather (batch_size, max_word_steps)\n",
    "        '''\n",
    "        rnn_inp = inputs[0]\n",
    "        ind_inp = inputs[1]\n",
    "        \n",
    "        ind_inp_zeroed = tf.select(tf.not_equal(ind_inp, -1), ind_inp, tf.zeros_like(ind_inp, dtype='int64'))\n",
    "        \n",
    "        def f(inp):\n",
    "            '''\n",
    "            Gathers the inds for the input mat of (max_char_len, char_lstm_dim)\n",
    "            '''\n",
    "            mat = inp[0]\n",
    "            inds = inp[1]\n",
    "            return tf.gather(mat, inds)\n",
    "        \n",
    "        map_fn_out = tf.map_fn(f, elems=(rnn_inp, ind_inp_zeroed), dtype='float32')\n",
    "        \n",
    "        return map_fn_out\n",
    "    \n",
    "    def get_output_shape_for(self, input_shape):\n",
    "        rnn_shape = input_shape[0]\n",
    "        ind_shape = input_shape[1]\n",
    "        return (rnn_shape[0], ind_shape[1], rnn_shape[2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 594,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "class SegmentLayer(Layer):\n",
    "    '''\n",
    "    Takes a segmented sum\n",
    "    '''\n",
    "    def __init__(self, seg_func_name='sum', **kwargs):\n",
    "        super(SegmentLayer, self).__init__(**kwargs)\n",
    "        if seg_func_name == 'sum':\n",
    "            self.seg_func = tf.segment_sum\n",
    "        elif seg_func_name == 'mean':\n",
    "            self.seg_func = tf.segment_mean\n",
    "        elif seg_func_name == 'max':\n",
    "            self.seg_func = tf.segment_max\n",
    "        else:\n",
    "            self.seg_func = tf.segment_sum\n",
    "        \n",
    "    def build(self, input_shape):\n",
    "        super(SegmentLayer, self).build(input_shape)\n",
    "    \n",
    "    def compute_mask(self, x, mask=None):\n",
    "        return K.cast(K.not_equal(x[1], -1), 'bool')\n",
    "    \n",
    "    def call(self, x, mask=None):\n",
    "        rnn_inp = x[0]\n",
    "        word_end_idx = x[1]\n",
    "        segment_mask = x[2]\n",
    "\n",
    "        # Need the max doc len\n",
    "        max_word_len = tf.shape(word_end_idx)[1]\n",
    "        \n",
    "        def f(inp):\n",
    "            '''\n",
    "            Performs a segmented sum on each input of (max_char_len, char_lstm_dim)\n",
    "            '''\n",
    "            mat = inp[0]\n",
    "            seg_mask = inp[1]\n",
    "            \n",
    "            # perform the segmented sum along the first axis\n",
    "            seg_sum = self.seg_func(mat, seg_mask)[:-1]  # don't want the last segment\n",
    "            \n",
    "            # need to pad the result such that we always have vectors of max_token_len\n",
    "            seg_sum_shape = tf.shape(seg_sum)\n",
    "            zero_pad = tf.zeros((max_word_len - seg_sum_shape[0], seg_sum_shape[1]), dtype='float32')\n",
    "            seg_sum_padded = tf.concat(0, [seg_sum, zero_pad])\n",
    "            \n",
    "            return seg_sum_padded\n",
    "        \n",
    "        map_fn_out = tf.map_fn(f, elems=(rnn_inp, segment_mask), dtype='float32')\n",
    "        \n",
    "        return map_fn_out\n",
    "    \n",
    "    def get_output_shape_for(self, input_shape):\n",
    "        rnn_shape = input_shape[0]\n",
    "        ind_shape = input_shape[1]\n",
    "        \n",
    "        return (rnn_shape[0], ind_shape[1], rnn_shape[2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Define a method for setting the models word_embedding layer to have pretrained embeddings\n",
    "def set_embeddings(embedding, weights, token_map):\n",
    "    '''\n",
    "    Takes in a gensim Word2Vec model and our keras model and then adapts the word_embeddings\n",
    "    in our model to the pre-trained vectors from the w2v model if they exist. Otherwise, they are\n",
    "    left to the original initalization\n",
    "    \n",
    "    Paramters\n",
    "    =========\n",
    "    w2v_model : gensim.models.word2vec.Word2Vec\n",
    "        Word2Vec model that is already loaded with pre-trained embedings\n",
    "    keras_model : keras.engine.training.Model\n",
    "        Keras model that has been constructed such that the word embedding layer has the same\n",
    "        number of embedding dimensions as the pre-trained embeddings\n",
    "    token_map : dict\n",
    "        map from token to index in the vocab\n",
    "    '''    \n",
    "    for token, ind in token_map.iteritems():\n",
    "        try:\n",
    "            pre_trained_emb = w2v_model[token]     \n",
    "        except:\n",
    "            pre_trained_emb = weights[ind]\n",
    "        weights[ind] = pre_trained_emb\n",
    "    return weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def construct_model(word_vocab_size, char_vocab_size,\n",
    "                    w_emb_dim=100, w_lstm_dim=128, \n",
    "                    c_emb_dim=100, c_lstm_dim=128, \n",
    "                    embedding=None, token2idx=None, trainable_embedding=True, dropout=0.5, \n",
    "                    char_emb=True):\n",
    "    '''\n",
    "    Constructs the NERNN in keras / tensorflow\n",
    "    \n",
    "    Parameters\n",
    "    ==========\n",
    "    word_vocab_size : int\n",
    "        size of the token vocabulary\n",
    "    char_vocab_size : int\n",
    "        size of the character vocabulary\n",
    "    w_emb_dim : int\n",
    "        size of token lstm layer\n",
    "    c_emb_dim : int\n",
    "        size of the char lstm layer\n",
    "    embedding : optional, gensim.model.word2vec.Word2Vec\n",
    "        pre-trained embeddings passed in the form of a gensim model\n",
    "    token2idx : optional, dict\n",
    "        map from tokens to indices\n",
    "    trainable_embedding : bool\n",
    "        indicates whether word embeddings are trainable\n",
    "    chat_emb : bool\n",
    "        indicates whether to use character embeddings\n",
    "    '''\n",
    "    \n",
    "    # =========================== #\n",
    "    # 1. Construct word Embedding #\n",
    "    # =========================== #\n",
    "\n",
    "    word_model_in = Input(shape=(None,), name='word_enc_in')\n",
    "    \n",
    "    if embedding:\n",
    "        \n",
    "        # Make some assertions\n",
    "        assert token2idx is not None, \"Must pass a map from token to idx for current training set with an embedding\"\n",
    "        assert w_emb_dim == embedding.vector_size, \"Word embedding dimension must be same as passed embedding\"\n",
    "        \n",
    "        # Set the weights to the pre-trained weights from the passed embedding\n",
    "        weights = np.random.uniform(-0.05, 0.05, size=(word_vocab_size+2, w_emb_dim))\n",
    "        weights = set_embeddings(embedding, weights, token2idx)\n",
    "        \n",
    "        # Construct the embedding layer\n",
    "        word_emb = Embedding(input_dim=word_vocab_size+2, output_dim=w_emb_dim, \n",
    "                             input_length=None, mask_zero=True, \n",
    "                             weights=[weights], name='word_embedding')\n",
    "        \n",
    "    else:\n",
    "        # Otherwise randomly initialize weights\n",
    "        word_emb = Embedding(input_dim=word_vocab_size+2, output_dim=w_emb_dim,\n",
    "                             input_length=None, mask_zero=True, name='word_embedding')\n",
    "\n",
    "    word_out = word_emb(word_model_in)\n",
    "    word_emb.trainable = trainable_embedding\n",
    "\n",
    "    # ============================== #\n",
    "    # 2. Construct the character RNN #\n",
    "    # ============================== #\n",
    "\n",
    "    if char_emb:\n",
    "        char_model_in = Input(shape=(None,), name='char_enc_in')\n",
    "        char_emb_out = Embedding(input_dim=char_vocab_size+2, output_dim=c_emb_dim, \n",
    "                                 input_length=None, mask_zero=True, name='char_embedding')(char_model_in)\n",
    "        char_out = Bidirectional(LSTM(c_lstm_dim, return_sequences=True, dropout_W=dropout))(char_emb_out)\n",
    "\n",
    "    # ====================================== #\n",
    "    # 3. Merge the Word RNN and the Char RNN #\n",
    "    # ====================================== #\n",
    "\n",
    "    # If we want to use character embeddings then extract the character representations from the\n",
    "    # character RNN\n",
    "    if char_emb:\n",
    "        # Create an input for the matrix of word end indices\n",
    "        inds = Input(shape=(None,), dtype='int64', name='token_end_idx')\n",
    "        \n",
    "        # Create an input for the matrix of segmentation masks\n",
    "        seg_mask = Input(shape=(None,), dtype='int64', name='char_seg_mask')\n",
    "\n",
    "        # Slice the character model out\n",
    "#         char_model_slice = GatherLayer()([temp_out, inds])\n",
    "        \n",
    "        # Segment the character model\n",
    "        char_model_seg = SegmentLayer(seg_func_name='mean')([char_out, inds, seg_mask])\n",
    "\n",
    "        # Concatenate the outputs of the word model and the sliced character model\n",
    "        merge_out = merge([word_out, char_model_seg], mode='concat', concat_axis=2)\n",
    "    else:\n",
    "        merge_out = word_out\n",
    "\n",
    "    # Add Bidirectional lstm here\n",
    "    gru_out = Bidirectional(LSTM(w_emb_dim, return_sequences=True, dropout_W=dropout))(merge_out)\n",
    "    \n",
    "    # ================== #\n",
    "    # 4. Compute Output  #\n",
    "    # ================== #\n",
    "\n",
    "    # Time distribute a final layers with binary output\n",
    "    hout = TimeDistributed(Dense(w_lstm_dim, activation='tanh'))(gru_out)\n",
    "    fout = TimeDistributed(Dense(1, activation='sigmoid'))(hout)\n",
    "    \n",
    "    model = Model([word_model_in, char_model_in, inds, seg_mask], output=[fout])\n",
    "    \n",
    "    # ====================================== #\n",
    "    # 5. Define Accuracy Metrics and Compile #\n",
    "    # ====================================== #\n",
    "\n",
    "    def accuracy(y_true, y_pred):\n",
    "        den = K.sum(K.cast(K.not_equal(y_true, -1), dtype='float32'))\n",
    "        num = K.sum(K.cast(K.equal(y_true, K.round(y_pred)) & K.not_equal(y_true, -1), dtype='float32'))\n",
    "        return num / den\n",
    "    \n",
    "    def true_pos(y_true, y_pred):\n",
    "        den = tf.reduce_sum(tf.cast(tf.equal(tf.round(y_pred), 1) & tf.not_equal(y_true, -1), dtype='float32'))\n",
    "        \n",
    "        i = tf.equal(y_true, tf.round(y_pred)) & \\\n",
    "                tf.equal(1., tf.round(y_pred)) & \\\n",
    "                tf.not_equal(y_true, -1)\n",
    "                \n",
    "        num = tf.reduce_sum(tf.cast(i, dtype='float32'))\n",
    "\n",
    "        frac = tf.select(den==0., 0., num / den)\n",
    "        \n",
    "        return frac\n",
    "    \n",
    "    model.compile(optimizer='adam', loss='binary_crossentropy', metrics=[accuracy, true_pos])\n",
    "    \n",
    "    return model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## b. Construct a gensim model that we can use to access underlying embeddings from the massive GoogleNews embedding matrix\n",
    "- 3 Million words in GoogleNews Vocab with 300 dimensional embeddings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Load in the gensim word2vec model\n",
    "w2vmodel = Word2Vec.load_word2vec_format('data/GoogleNews-vectors-negative300.bin', binary=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# II. Methods to Bring in Wikipedia Data to Train On\n",
    "- Downloaded WikiNER data into data directory\n",
    "- Download WikiGold data into data directory\n",
    "- Will read in the whole WikiNER dataset into one pandas df\n",
    "- Will read in pre-split train/test/dev data into pandas for WikiGold\n",
    "- Preprocess the data to only tag PER (peoples names)\n",
    "- Will batch sentences together into 'Documents' for WikiGold"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Utility Functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 339,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import re\n",
    "\n",
    "def untokenize(words):\n",
    "    \"\"\"\n",
    "    Untokenizing a text undoes the tokenizing operation, restoring\n",
    "    punctuation and spaces to the places that people expect them to be.\n",
    "    Ideally, `untokenize(tokenize(text))` should be identical to `text`,\n",
    "    except for line breaks.\n",
    "    \"\"\"\n",
    "    text = ' '.join(words)\n",
    "    step1 = text.replace(\"`` \", '\"').replace(\" ''\", '\"').replace('. . .',  '...')\n",
    "    step2 = step1.replace(\" ( \", \" (\").replace(\" ) \", \") \")\n",
    "    step3 = re.sub(r' ([.,:;?!%]+)([ \\'\"`])', r\"\\1\\2\", step2)\n",
    "    step4 = re.sub(r' ([.,:;?!%]+)$', r\"\\1\", step3)\n",
    "    step5 = step4.replace(\" '\", \"'\").replace(\" n't\", \"n't\").replace(\n",
    "         \"can not\", \"cannot\")\n",
    "    step6 = step5.replace(\" ` \", \" '\")\n",
    "    return step6.strip()\n",
    "\n",
    "def prepend_zeros(arr, num_zeros=1, dtype='int64'):\n",
    "    '''\n",
    "    Takes a 1-D numpy array and prepends the specified number of zeros\n",
    "    '''\n",
    "    zs = np.zeros(shape=(num_zeros,), dtype=dtype)\n",
    "    return np.concatenate([zs, arr], axis=0)\n",
    "\n",
    "def spans(txt, tokens):\n",
    "    '''\n",
    "    Takes the original (read: \"untokenized\" text) and the tokens and returns a list of word\n",
    "    end indices.\n",
    "    \n",
    "    Parameters\n",
    "    ==========\n",
    "    txt : string\n",
    "        untokenized / raw string we want to index the tokens into\n",
    "    tokens : list, array\n",
    "        list of tokens that make up the txt\n",
    "        \n",
    "    Returns\n",
    "    =======\n",
    "    word_inds : list\n",
    "        list of word span indices for the tokens into the txt\n",
    "    '''\n",
    "    word_inds = []\n",
    "    offset = 0\n",
    "    for token in tokens:\n",
    "        offset = txt.find(token, offset)\n",
    "        word_inds.append((offset, offset+len(token)))\n",
    "        offset += len(token)\n",
    "    return word_inds\n",
    "\n",
    "def segmentation_mask(word_end_inds):\n",
    "    '''\n",
    "    Takes end (inclusive) index of the tokens and returns a mask with length num of characters\n",
    "    that masks each token with a monotonically increasing index\n",
    "    \n",
    "    Example\n",
    "    -------\n",
    "    ['H','e','l','l','o',' ','W','o','r','l','d'] -> [0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1]\n",
    "    '''\n",
    "    if len(word_end_inds) == 0:\n",
    "        return np.empty((0,), dtype='int64')\n",
    "    doc_len = max(word_end_inds) + 1\n",
    "    \n",
    "    word_inds = [-1] + word_end_inds\n",
    "    \n",
    "    seg_mask = np.zeros(doc_len, dtype='int64')\n",
    "    \n",
    "    for i in range(len(word_inds) - 1):\n",
    "        s = word_inds[i]\n",
    "        e = word_inds[i+1]\n",
    "        seg_mask[s+1:e+1] = i\n",
    "    return seg_mask "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## a. Methods for Reading in Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 432,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# ==== #\n",
    "# Vars #\n",
    "# ==== #\n",
    "\n",
    "# Paths\n",
    "DATA_DIR = './data'\n",
    "WIKIGOLD_DATA = os.path.join(DATA_DIR, 'WikiGold')\n",
    "WIKINER_DATA = os.path.join(DATA_DIR, 'raw', 'aij-wikiner-en-wp3')\n",
    "\n",
    "# Misc.\n",
    "DOCSTART_TAG = '-DOCSTART-'\n",
    "\n",
    "# ======================================= #\n",
    "# Methods for Reading and Processing Data #\n",
    "# ======================================= #\n",
    "\n",
    "# Read in the data only (no processing except putting in in pandas)\n",
    "def read_wikigold_datasets(data_dir):\n",
    "    '''\n",
    "    Reads in the wikipedia Gold datasets from the data directory.\n",
    "    These datasets were extracted using scripts in\n",
    "    https://github.palantir.build/DeltaSierra/tagtrex\n",
    "    \n",
    "    Paramters\n",
    "    =========\n",
    "    data_dir : str\n",
    "        path to the data directory\n",
    "        \n",
    "    Returns\n",
    "    =======\n",
    "    train_df : pandas.core.frame.DataFrame\n",
    "        dataframe with training data\n",
    "    test_df : pandas.core.frame.DataFrame\n",
    "        dataframe with test data\n",
    "    dev_df : pandas.core.frame.DataFrame\n",
    "        dataframe with dev data\n",
    "    TODO: (areiner) what the hell is dev data?... I didn't do these splits\n",
    "    '''\n",
    "    \n",
    "    dataset_names = ('train', 'test', 'dev')\n",
    "    datasets = []  # ordered train, test, dev\n",
    "    for dname in dataset_names: \n",
    "        found_datasets = glob.glob(data_dir + '/*' + dname + '*.pkl')\n",
    "        if len(found_datasets)==0:\n",
    "            print \"No dataset with name {} found\".format(dname)\n",
    "        elif len(found_datasets) > 1:\n",
    "            print \"Multiple dataset with name {} found\".format(dname)\n",
    "        else:\n",
    "            df = pd.read_pickle(found_datasets[0])\n",
    "            datasets.append(df)\n",
    "    return datasets\n",
    "\n",
    "def read_wikiner_dataset(data_path):\n",
    "    '''\n",
    "    Reads in wikiner data in its raw text form and outputs a dataframe\n",
    "    '''\n",
    "    # Read in each line and process it\n",
    "    with open(data_path, 'r') as f:\n",
    "        \n",
    "        # Track the list of tokens, tags\n",
    "        token_lists = []\n",
    "        tags_lists = []\n",
    "        \n",
    "        for line in f:\n",
    "            # Current sample tokens / tags\n",
    "            token_list = []\n",
    "            tag_list = []\n",
    "            \n",
    "            # Processing\n",
    "            l = line.strip()\n",
    "            aug_tokens = l.split(' ')\n",
    "            \n",
    "            # Discard the POS\n",
    "            for at in aug_tokens:\n",
    "                at_split = at.split('|')\n",
    "                token_list.append(at_split[0])\n",
    "                tag_list.append(at_split[-1])\n",
    "            \n",
    "            # Add the sample\n",
    "            token_lists.append(token_list)\n",
    "            tags_lists.append(tag_list)\n",
    "            \n",
    "    df = pd.DataFrame({'document': token_lists[1:], 'tags': tags_lists[1:]})\n",
    "    return df\n",
    "    \n",
    "# Method to filter all tags such that it is only 'O' or 'PER'\n",
    "def modify_tags(df):\n",
    "    '''\n",
    "    Takes in a dataframe with columns \"sentence\" and \"tags\" and modifies\n",
    "    the \"tags\" column to convert everything that is not \"B-PER\" to \"O\" and\n",
    "    converts \"B-PER\" to \"PER\"\n",
    "    \n",
    "    Parameters\n",
    "    ==========\n",
    "    df : pandas.core.frame.DataFrame\n",
    "        input dataframe of sentences and tags (with columns \"sentence\" and \"tags\")\n",
    "    '''\n",
    "    def modify_taglist(taglist):\n",
    "        '''\n",
    "        Function to apply to each array of tags in each cell of the \"tags\"\n",
    "        column in the dataframe\n",
    "        '''\n",
    "        m = {'B-PER': 'PER', 'I-PER': 'PER'}\n",
    "        return map(lambda v: m.get(v, 'O'), taglist)\n",
    "\n",
    "    df['tags'] = df['tags'].apply(modify_taglist)\n",
    "    \n",
    "    return df\n",
    "\n",
    "# Method to find the index for document separations\n",
    "def calc_docstart_inds(df):\n",
    "    '''\n",
    "    Takes in a dataframe and looks for specific document start tags\n",
    "    '''\n",
    "    return df.index[df.sentence.map(lambda x: DOCSTART_TAG in x)].get_values()\n",
    "\n",
    "# Method to concatenate sentence tags to document tags\n",
    "def construct_doc_tag(dfs, max_sent_per_doc=None):\n",
    "    '''\n",
    "    Takes a DataFrame slice and returns a dataframe of tokens and tags that\n",
    "    concatenates all sentence tokens and tags for the whole dataframe, potentially\n",
    "    into groups of size specified by max_sent_per_doc size\n",
    "    '''\n",
    "    new_df = pd.DataFrame(columns=['document', 'tags'])\n",
    "    \n",
    "    sentences = []\n",
    "    tags = []\n",
    "    \n",
    "    for i, (_, row) in enumerate(dfs.iterrows()):\n",
    "        \n",
    "        if (max_sent_per_doc is not None) and i!=0 and (i % max_sent_per_doc == 0):\n",
    "            new_df = new_df.append({'document': sentences, 'tags': tags}, ignore_index=True)\n",
    "            sentences = []\n",
    "            tags = []\n",
    "\n",
    "        tokens = row.sentence\n",
    "        tagseq = row.tags\n",
    "        \n",
    "        sentences.extend(tokens)\n",
    "        tags.extend(tagseq)\n",
    "    \n",
    "    new_df = new_df.append({'document': sentences, 'tags': tags}, ignore_index=True)\n",
    "    \n",
    "    return new_df\n",
    "\n",
    "# Method to cluster sentence token dataframes into documents\n",
    "def transform_sent_to_docs(df, max_sent_per_doc=None):\n",
    "    '''\n",
    "    Transform a DataFrame of lists of tokens and tags per sentence into\n",
    "    a DataFrame of \"Documents\" and the tags for that document.\n",
    "    At a minimum, we split by document length as per the wikipedia page\n",
    "    '''\n",
    "    doc_df = pd.DataFrame(columns=['document', 'tags'])\n",
    "    \n",
    "    # 1. Calculate the indices of Document starts\n",
    "    doc_starts = prepend_zeros(calc_docstart_inds(df))\n",
    "    doc_starts = np.concatenate((doc_starts, [len(df)]), axis=0)\n",
    "    \n",
    "    # 2. Slice the df for the sentences in each document\n",
    "    for i, (start, end) in enumerate(zip(doc_starts[:-1], doc_starts[1:])):\n",
    "        if i != 0:\n",
    "            start += 1\n",
    "        dfs = df.iloc[start:end]\n",
    "        inc_doc_df = construct_doc_tag(dfs, max_sent_per_doc=max_sent_per_doc)\n",
    "        doc_df = doc_df.append(inc_doc_df)\n",
    "    \n",
    "    doc_df.reset_index(inplace=True, drop=True)\n",
    "    \n",
    "    return doc_df\n",
    "        \n",
    "# Method that adds a column which contains the untokenized sentence / document\n",
    "def untokenize_column(df, col='document', new_col='document_string'):\n",
    "    '''\n",
    "    Untokenizes a column of lists of tokens. The intended use in the pipeline is to\n",
    "    construct untokenized strings after clustering sentences into documents or partial\n",
    "    documents.\n",
    "    '''\n",
    "    df[new_col] = df[col].map(untokenize)\n",
    "    return df\n",
    "\n",
    "# Method that adds a column which contains the index of the tokens into the untokenized sentence / document\n",
    "def word_index_columns(df, doc_col='document_string', tok_col='document'):\n",
    "    '''\n",
    "    Takes a column of strings (either sentences or documents as long as it's one continuous string) \n",
    "    and creats two new columns a 'word_start_inds' column that contains the start indices of all tokens\n",
    "    and a 'word_end_inds' that contains the end indices of all tokens\n",
    "    '''\n",
    "    # Calculate the word spanning indices\n",
    "    word_inds = df.apply(lambda r: spans(r[doc_col], r[tok_col]), axis=1)\n",
    "    word_start_inds = word_inds.apply(lambda v: [e[0] for e in v])\n",
    "    word_end_inds = word_inds.apply(lambda v: [e[1] - 1 for e in v])\n",
    "    word_segment_mask = word_end_inds.map(segmentation_mask)\n",
    "    \n",
    "    # Insert new columns\n",
    "    df['word_start_inds'] = word_start_inds\n",
    "    df['word_end_inds'] = word_end_inds\n",
    "    df['word_segment_mask'] = word_segment_mask\n",
    "    return df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## b. Methods for data preprocessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 431,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Construct word and character maps\n",
    "def construct_map(element_lists, vocab_size=None):\n",
    "    '''\n",
    "    Constructs a vocabulary from \n",
    "    '''\n",
    "    from collections import Counter\n",
    "    c = Counter()\n",
    "    for els in element_lists:\n",
    "        c.update(els)\n",
    "\n",
    "    if vocab_size is not None:\n",
    "        most_common = [x[0] for x in c.most_common(vocab_size)]\n",
    "        hash_map = dict(zip(most_common, range(1, len(most_common)+1)))\n",
    "    else:\n",
    "        hash_map = dict(zip(c.keys(), range(1, len(c)+1)))\n",
    "\n",
    "    return hash_map\n",
    "\n",
    "def reverse_map(m):\n",
    "    m_inv = dict(((ind, k) for k, v in m.iteritems()))\n",
    "    return m_inv\n",
    "    \n",
    "def character_column(df):\n",
    "    '''\n",
    "    Inserts a column of the individual characters into the dataframe \n",
    "    \n",
    "    Parameters\n",
    "    ==========\n",
    "    df : pandas.core.frame.DataFrame\n",
    "        dataframe containing the string / untokenized documents\n",
    "        \n",
    "    Returns\n",
    "    =======\n",
    "    new_df : pandas.core.frame.DataFrame\n",
    "        Updated df\n",
    "    '''\n",
    "    df['chars'] = df['document_string'].map(lambda v: list(v))\n",
    "    return df\n",
    "\n",
    "def construct_word_char_maps(df, vocab_size=None, return_inv_dicts=False):\n",
    "    '''\n",
    "    Construct the word and character maps from the dataframe\n",
    "    \n",
    "    Parameters\n",
    "    ==========\n",
    "    df : pandas DataFrame\n",
    "        dataframe with tokens and character lists already constructed\n",
    "    \n",
    "    Returns\n",
    "    =======\n",
    "    token_map : dict\n",
    "    char_map : dict\n",
    "    '''\n",
    "    token_map = construct_map(df['document'])\n",
    "    char_map = construct_map(df['chars'])\n",
    "    \n",
    "    if return_inv_dicts:\n",
    "        token_map_inv = reverse_map(token_map)\n",
    "        char_map_inv = reverse_map(char_map)\n",
    "        \n",
    "        return token_map, char_map, token_map_inv, char_map_inv\n",
    "    else:\n",
    "        return token_map, char_map\n",
    "\n",
    "def encode_strings(element_lists, hash_map):\n",
    "    '''\n",
    "    Encode the element_list in terms of integers\n",
    "    NOTE: 0 is reserved for masking\n",
    "    '''\n",
    "    new_element_list = []\n",
    "    for els in element_lists:\n",
    "        new_els = map(lambda x: hash_map.get(x, len(hash_map)+1), els)\n",
    "        new_element_list.append(new_els)\n",
    "    return new_element_list\n",
    "    \n",
    "def encode_tokens_chars(df, token_map, char_map):\n",
    "    '''\n",
    "    Encode the tokens and characters\n",
    "    '''\n",
    "    df['token_enc'] = encode_strings(df['document'], token_map)\n",
    "    df['char_enc'] = encode_strings(df['chars'], char_map)\n",
    "    return df\n",
    "\n",
    "def _encode_tags(tag_list):\n",
    "    '''\n",
    "    Encode a taglist\n",
    "    '''\n",
    "    def _tag_str_to_int(tag_str):\n",
    "        if tag_str == 'PER':\n",
    "            return 1\n",
    "        else:\n",
    "            return 0\n",
    "        \n",
    "    tags_enc = map(_tag_str_to_int, tag_list)\n",
    "    return tags_enc\n",
    "    \n",
    "def encode_tags(df):\n",
    "    '''\n",
    "    Encode the tags as binary outcomes\n",
    "    \n",
    "    Parameters\n",
    "    =========\n",
    "    df : pandas.core.frame.DataFrame\n",
    "        dataframe with character tags that are in the set {'O', and 'PER'}\n",
    "    '''\n",
    "    \n",
    "    df['tags_enc'] = df['tags'].apply(_encode_tags)\n",
    "    return df\n",
    "\n",
    "def remove_pure_neg(df, frac=0.1):\n",
    "    '''\n",
    "    Data processing function that removes a fraction of the samples that contain no positive tags\n",
    "    '''\n",
    "    \n",
    "    if frac == 0.:\n",
    "        return df\n",
    "    \n",
    "    # Extract the samples with positive tags and a fraction of those with no positive tags\n",
    "    nonz_idx = df.index[df.tags_enc.map(sum)!=0]\n",
    "    z_idx = df.index[df.tags_enc.map(sum)==0]\n",
    "    z_idx_red = z_idx[:int(len(z_idx)*(1-frac))]\n",
    "    new_idx = z_idx_red.append(nonz_idx)\n",
    "    \n",
    "    # Select the new index and shuffle the samples\n",
    "    df_new = df.loc[new_idx].sample(frac=1.).reset_index()\n",
    "    \n",
    "    return df_new\n",
    "    \n",
    "def _construct_character_tags(word_start_inds, word_end_inds, char_list, tag_enc):\n",
    "    '''\n",
    "    \n",
    "    '''\n",
    "    char_tags = np.zeros(len(char_list), dtype='int64')\n",
    "    \n",
    "    word_start_inds = np.array(word_start_inds)\n",
    "    word_end_inds = np.array(word_end_inds)\n",
    "    tag_enc = np.array(tag_enc)\n",
    "    \n",
    "    word_start_ind_tag = word_start_inds[tag_enc==1]\n",
    "    word_end_ind_tag = word_end_inds[tag_enc==1]\n",
    "    word_span_tag = zip(word_start_ind_tag, word_end_ind_tag)\n",
    "    \n",
    "    for (s, e) in word_span_tag:\n",
    "        char_tags[s:e+1] = 1\n",
    "    \n",
    "    return list(char_tags)\n",
    "\n",
    "def construct_character_tags(df):\n",
    "    df['char_tag_mask'] = df.apply(lambda r: _construct_character_tags(r.word_start_inds, r.word_end_inds, \n",
    "                                                                       r.chars, r.tags_enc), axis=1)\n",
    "    return df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## c. Full data processing pipeline function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 513,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# ======================== #\n",
    "# Boilerplate Augmentation #\n",
    "# ======================== #\n",
    "def augment_dataset(df, doc_splits=False, max_sent_per_doc=None):\n",
    "    '''\n",
    "    Processes a pandas dataframe with a columns\n",
    "    \n",
    "    Parameters\n",
    "    ==========\n",
    "    df : pandas DataFrame\n",
    "        dataframe containing columns \"document\" and \"tags\"\n",
    "    \n",
    "    doc_splits : bool\n",
    "        boolean indicating whether the dataframe contains document segmentation tags\n",
    "        \n",
    "    max_sent_per_doc : None, or int\n",
    "        only applicable if doc_splits is True. Indicates the maximum number of sentences per doc.\n",
    "        If None and doc_splits is True then will aggregate by document with no restriction on the number\n",
    "        of sentences that compose it\n",
    "        \n",
    "    Returns\n",
    "    =======\n",
    "    df : pandas DataFrame\n",
    "        dataframe augmented with\n",
    "            - tags are processed (only processing step. The other steps are augmenting)\n",
    "            - \"document_string\" (by untokenize_column)\n",
    "            - \"word_start_inds\", \"word_end_inds\", \"word_segment_mask\" (by word_index_columns)\n",
    "            - \"chars\" (by character_column)\n",
    "    '''\n",
    "    df = modify_tags(df)\n",
    "    if doc_splits:\n",
    "        df = transform_sent_to_docs(df, max_sent_per_doc=max_sent_per_doc) # Concatenate sentence tokens into topics\n",
    "    else:\n",
    "        if max_sent_per_doc:\n",
    "            print \"Specified max_sentences_per doc but doc_splits is false\"\n",
    "    df = untokenize_column(df)  # (approximately) concatenate the tokens into documents\n",
    "    df = word_index_columns(df)  # index the tokens into the untokenized strings\n",
    "    df = character_column(df)\n",
    "    return df\n",
    "\n",
    "def encode_dataset(df, maps=None):\n",
    "    '''\n",
    "    Encode the dataset\n",
    "    \n",
    "    Parameters\n",
    "    ==========\n",
    "    df : pandas DataFrame\n",
    "        dataframe containing \"document\", \"chars\", and \"tags\" columns\n",
    "        \n",
    "    Returns\n",
    "    =======\n",
    "    df : pandas DataFrame\n",
    "        dataframe augmented with encoding columns\n",
    "            - \"token_enc\"\n",
    "            - \"char_enc\"\n",
    "            - \"tags_enc\"\n",
    "    '''\n",
    "    if maps is None:\n",
    "        tm, cm = construct_word_char_maps(df)  # first construct the hash maps\n",
    "    else:\n",
    "        tm, cm = maps\n",
    "    df = encode_tokens_chars(df, tm, cm)  # encode the tokens and the characters\n",
    "    df = encode_tags(df)  # encode the tags as binary outcomes 'O' -> 0 and 'PER' -> 1\n",
    "    \n",
    "    return df, tm, cm\n",
    "\n",
    "# ================== #\n",
    "# WikiGold Pipelines #\n",
    "# ================== #\n",
    "\n",
    "def process_wikigold_dataset(df, max_sent_per_doc=None, rem_neg_frac=0.0, maps=None):\n",
    "    \n",
    "    # Text Processing\n",
    "    df = augment_dataset(df, doc_splits=True, max_sent_per_doc=max_sent_per_doc)\n",
    "\n",
    "    # Encoding work\n",
    "    df, tm, cm = encode_dataset(df, maps=maps)\n",
    "    \n",
    "    # Filtering negative samples\n",
    "    df = remove_pure_neg(df, frac=rem_neg_frac)  # remove a fraction of negative samples\n",
    "    \n",
    "    # Extra info for model validation\n",
    "    df = construct_character_tags(df)\n",
    "    \n",
    "    return df, tm, cm\n",
    "\n",
    "def run_wikigold_data_pipeline(data_path, combine=True, max_sent_per_doc=None, rem_neg_frac=0.0, maps=None):\n",
    "    '''\n",
    "    Runs the entire transformation pipeline for WikiGold data.\n",
    "    \n",
    "    Pipeline Steps\n",
    "    --------------\n",
    "    1. Read in wiki datasets (train, test, dev)\n",
    "    For each dataset in the wiki datasets:\n",
    "        2. Modify the IOB tags to only 'O' and 'PER'\n",
    "        3. Take sentence examples and concatenate them into individual documents\n",
    "        4. Take the documents which consist of lists of tokens and \"untokenize\" them into continuous strings\n",
    "        5. Take the tokens and index them into the untokenized text so we have start and end indices for each token\n",
    "        6. Split up the untokenized text into a list of characters\n",
    "        7. Construct element -> monotonically increasing index map for both tokens and characters\n",
    "        8. Encode the tokens and characters by their index in their respective maps\n",
    "        9. Encode the tags as binary outcomes 'O' -> 0 and 'PER' -> 1\n",
    "    \n",
    "    Parameters\n",
    "    ==========\n",
    "    data_path : str\n",
    "        path to the directory that contains the WikiGold datasets. These are the datasets that are a result\n",
    "        of Stan's (tagtrex guy's) processing steps (with slight modification that he takes out DOCSTART tags that\n",
    "        indicate when a wiki article begins and I don't)\n",
    "    \n",
    "    combine : bool\n",
    "        indicates whether to combine the pre-split \"train\", \"test\", \"dev\" training sets or to leave them out\n",
    "    '''\n",
    "    # 1. Reading in data\n",
    "    wiki_datasets = read_wikigold_datasets(data_path)\n",
    "\n",
    "    if not combine:\n",
    "    \n",
    "        wiki_datasets_processed = []\n",
    "        wiki_datasets_maps = []\n",
    "\n",
    "        # 2. Run processing on each dataframe\n",
    "        for df in wiki_datasets:\n",
    "            new_df, token_map, char_map = process_wikigold_dataset(\n",
    "                df,\n",
    "                max_sent_per_doc=max_sent_per_doc,\n",
    "                rem_neg_frac=rem_neg_frac,\n",
    "                maps=maps\n",
    "            )\n",
    "            wiki_datasets_processed.append(new_df)\n",
    "            wiki_datasets_maps.append((token_map, char_map))\n",
    "    else:\n",
    "        # Need to append a placeholder DOCSTART indicator for all but the first dataframe\n",
    "        for i in range(1, len(wiki_datasets)):\n",
    "            doc_bookmark = pd.DataFrame({'sentence': [[DOCSTART_TAG]], 'tags': [['O']]})\n",
    "            wiki_datasets[i] = pd.concat([doc_bookmark, wiki_datasets[i]], ignore_index=True)\n",
    "            if wiki_datasets[i].sentence.iloc[-1] == [DOCSTART_TAG]:\n",
    "                wiki_datasets[i] = wiki_datasets[i][:-1]\n",
    "\n",
    "        wiki_comb = pd.concat(wiki_datasets, ignore_index=True)\n",
    "        new_df, token_map, char_map = process_wikigold_dataset(\n",
    "            wiki_comb,\n",
    "            max_sent_per_doc=max_sent_per_doc,\n",
    "            rem_neg_frac=rem_neg_frac,\n",
    "            maps=maps\n",
    "        )\n",
    "        wiki_datasets_processed = new_df\n",
    "        wiki_datasets_maps = (token_map, char_map)\n",
    "    \n",
    "    return wiki_datasets_processed, wiki_datasets_maps\n",
    "\n",
    "# ================= #\n",
    "# WikiNER Pipelines #\n",
    "# ================= #\n",
    "\n",
    "def process_wikiner_dataset(df, rem_neg_frac=0.0, maps=None):\n",
    "    # Text Processing\n",
    "    df = augment_dataset(df)\n",
    "\n",
    "    # Encoding work\n",
    "    df, tm, cm = encode_dataset(df, maps=maps)\n",
    "    df = remove_pure_neg(df, frac=rem_neg_frac)  # remove a fraction of negative tags\n",
    "\n",
    "    # Extra info for model validation\n",
    "    df = construct_character_tags(df)\n",
    "    \n",
    "    return df, tm, cm\n",
    "\n",
    "def run_wikiner_data_pipeline(data_path, rem_neg_frac=0.0, maps=None):\n",
    "    df = read_wikiner_dataset(data_path)\n",
    "    df, tm, cm = process_wikiner_dataset(df, rem_neg_frac=rem_neg_frac, maps=maps)\n",
    "    return df, tm, cm"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## d. Regularization Methods"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 263,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def unk_insertion(token_enc, tags_enc, unk_idx, unk_neg_prob=0.02, unk_pos_prob=0.1):\n",
    "    '''\n",
    "    Imparts unknown tags into the dataset\n",
    "    '''\n",
    "    \n",
    "    # If the probablities are set to None then there is no unknown insertion\n",
    "    if (unk_neg_prob is None) and (unk_pos_prob is None):\n",
    "        return token_enc\n",
    "    \n",
    "    # Get a selection mask for the negative and positive tags\n",
    "    tags_neg = tags_enc == 0\n",
    "    tags_pos = tags_enc == 1\n",
    "    unk_neg_replace_mask = (np.random.rand(token_enc.shape[0], token_enc.shape[1]) < unk_neg_prob) & tags_neg\n",
    "    unk_pos_replace_mask = (np.random.rand(token_enc.shape[0], token_enc.shape[1]) < unk_pos_prob) & tags_pos\n",
    "    \n",
    "    token_enc_unk = token_enc[:]\n",
    "    token_enc_unk[unk_neg_replace_mask] = unk_idx\n",
    "    token_enc_unk[unk_pos_replace_mask] = unk_idx\n",
    "    \n",
    "    return token_enc_unk"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## e. Padding and Generating Training Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 429,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# ================= #\n",
    "# Padding Functions #\n",
    "# ================= #\n",
    "\n",
    "def pad_encoding_column(col, value=0):\n",
    "    '''\n",
    "    Takes a pandas series of lists of encodings and returns a single matrix with padded results\n",
    "    '''    \n",
    "    col_padded = sequence.pad_sequences(col.tolist(), padding='post', value=value)\n",
    "    return col_padded\n",
    "\n",
    "def segment_mask_padding(seg_col, max_doc_len):\n",
    "    col_pad = pad_encoding_column(seg_col, value=-1)\n",
    "    \n",
    "    max_char_len = col_pad.shape[1]\n",
    "    \n",
    "    for i, r in enumerate(col_pad):\n",
    "        last_word_idx = max(r)\n",
    "        idx_last_word = np.argmax(r)\n",
    "        mask_inds = (r == -1)\n",
    "        \n",
    "        # from idx of last mask + 1 put enough unique values so there are as many unique values as \n",
    "        # the maximum number of unique tokens\n",
    "        total_pad = np.zeros(mask_inds.sum(), dtype='int64')\n",
    "        fake_unique = np.arange(last_word_idx + 1, max_doc_len)\n",
    "        total_pad[:len(fake_unique)] = fake_unique\n",
    "        \n",
    "        # then fill in the remaing open slots with the last value\n",
    "        total_pad[len(fake_unique):] = fake_unique[:][-1:]\n",
    "        r[mask_inds] = total_pad\n",
    "    return col_pad\n",
    "\n",
    "def segment_mask_padding2(seg_col):\n",
    "    col_pad = pad_encoding_column(seg_col, value=-1)\n",
    "    \n",
    "    for r in col_pad:\n",
    "        last_word_idx = max(r)\n",
    "        mask_inds = (r == -1)\n",
    "        \n",
    "        r[mask_inds] = last_word_idx + 1\n",
    "    return col_pad\n",
    "\n",
    "# =============== #\n",
    "# Data Generation #\n",
    "# =============== #\n",
    "\n",
    "def generate_padded_data(df):\n",
    "    '''\n",
    "    Take the dataframe and return numpy matrices with padded encodings\n",
    "    '''\n",
    "    cols_to_pad = ['token_enc', 'char_enc', 'tags_enc', \n",
    "                   'word_start_inds', 'word_end_inds', 'word_segment_mask']\n",
    "    padded_data = []\n",
    "    max_doc_len = max(df.document.map(len))\n",
    "\n",
    "    for col in cols_to_pad:\n",
    "        if ('tag' in col) or ('inds' in col):\n",
    "            col_pad = pad_encoding_column(df[col], value=-1)\n",
    "        elif 'segment' in col:\n",
    "            col_pad = segment_mask_padding2(df[col])\n",
    "        else:\n",
    "            col_pad = pad_encoding_column(df[col])\n",
    "        padded_data.append(col_pad)\n",
    "    return padded_data\n",
    "\n",
    "def data_generator(df, batch_size=20, nb_epoch=10, shuffle=True, unk_neg_prob=None, unk_pos_prob=None):\n",
    "    '''\n",
    "    A python generator that yields shuffled batches\n",
    "    '''\n",
    "    # Generate the full padded numpy arrays from the dataframe\n",
    "    token_enc, char_enc, tags_enc, word_start_ind, word_end_ind, word_segment_mask = generate_padded_data(df)\n",
    "    \n",
    "    for _ in range(nb_epoch):\n",
    "        \n",
    "        # Apply random unk insertion\n",
    "        if (unk_neg_prob is not None) or (unk_pos_prob is not None):\n",
    "            unk_idx = df.token_enc.map(max).max() + 1\n",
    "            token_enc_unk = unk_insertion(token_enc, tags_enc, unk_idx,\n",
    "                                          unk_neg_prob=unk_neg_prob, unk_pos_prob=unk_pos_prob)\n",
    "            vars = [token_enc_unk, char_enc, tags_enc, word_start_ind, word_end_ind, word_segment_mask]\n",
    "        else:\n",
    "            vars = [token_enc, char_enc, tags_enc, word_start_ind, word_end_ind, word_segment_mask]\n",
    "\n",
    "        # Shuffle the index\n",
    "        idx = np.arange(token_enc.shape[0])\n",
    "        if shuffle:\n",
    "            np.random.shuffle(idx)\n",
    "\n",
    "        # Determine number of batches in the epoch\n",
    "        num_batches = (token_enc.shape[0] / batch_size) + 1\n",
    "\n",
    "        for i in range(num_batches):\n",
    "            idx_batch = idx[i*batch_size:(i+1)*batch_size]\n",
    "            token_enc_batch, char_enc_batch, \\\n",
    "            tags_enc_batch, word_start_ind_batch, \\\n",
    "            word_end_ind_batch, word_segment_mask_batch = map(lambda v: v[idx_batch], vars)\n",
    "\n",
    "            inp = [token_enc_batch, char_enc_batch, word_end_ind_batch, word_segment_mask_batch]\n",
    "            target = np.expand_dims(tags_enc_batch, 2)\n",
    "            yield (inp, target)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# III. Load Data, Construct Model and Bring in Pre-trained word emebddings\n",
    "- Will load WikiNER as primary training set and then will load WikiGold for testing\n",
    "- For now, we will use GoogleNews embedding vectors with dimensionality 300\n",
    "- We will not pre-train character emebeddings"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Data and Training Vars"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 536,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "MODEL_DIR = \"models\"  # top level model directory\n",
    "MODEL_WEIGHTS_DIR = \"models/\"  # target directory that contains weights for current model version"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## a. Load WikiNER and WikiGold Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 281,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Load WikiNER datasets\n",
    "wikiner, tm_ner, cm_ner = run_wikiner_data_pipeline(\n",
    "    WIKINER_DATA, \n",
    "    rem_neg_frac=0.4\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 522,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Load WikiGold datasets\n",
    "wikigold, (tm_wg, cm_wg) = run_wikigold_data_pipeline(\n",
    "    WIKIGOLD_DATA,\n",
    "    combine=True,\n",
    "    max_sent_per_doc=1,\n",
    "    rem_neg_frac=0.0, \n",
    "    maps=(tm_ner, cm_ner)\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 540,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Padded data for WikiNer\n",
    "token_enc_ner, char_enc_ner, \\\n",
    "tags_enc_ner, word_start_ind_ner, \\\n",
    "word_end_ind_ner, word_segment_mask_ner = generate_padded_data(wikiner)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 529,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Padded data for WikiGold\n",
    "token_enc_wg, char_enc_wg, \\\n",
    "tags_enc_wg, word_start_ind_wg, \\\n",
    "word_end_ind_wg, word_segment_mask_wg = generate_padded_data(wikigold)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 595,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Training vocab sizes\n",
    "word_vocab_size = len(tm_ner)\n",
    "char_vocab_size = len(cm_ner)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Local Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 543,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Local test model construction\n",
    "test_model = construct_model(word_vocab_size=word_vocab_size, char_vocab_size=char_vocab_size,\n",
    "                        w_emb_dim=5, w_lstm_dim=16, c_emb_dim=5, c_lstm_dim=8, trainable_embedding=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Real Model (Cloud)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Read model construction\n",
    "model = construct_model(word_vocab_size=word_vocab_size, char_vocab_size=char_vocab_size,\n",
    "                        w_emb_dim=300, w_lstm_dim=100, c_emb_dim=50, c_lstm_dim=100, \n",
    "                        embedding=w2vmodel, token2idx=tm_ner, trainable_embedding=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# VI. Train the Model!"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Utility loading methods"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 535,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# ======================================== #\n",
    "# Getting Weights and Archive Folder Names #\n",
    "# ======================================== #\n",
    "\n",
    "def all_weights_in_dir(d):\n",
    "    return glob.glob(os.path.join(d, '*.hdf5'))\n",
    "\n",
    "def best_weights(path):\n",
    "    '''\n",
    "    Returns the file name for the best (most recent weights)\n",
    "    '''\n",
    "    all_weights = all_weights_in_dir(path)\n",
    "    if len(all_weights) == 0:\n",
    "        print \"No weights found!\"\n",
    "        return\n",
    "    return max(all_weights, key = os.path.getctime)\n",
    "\n",
    "def get_archive_folders():\n",
    "    return glob.glob(MODEL_DIR + '/weights_v*')\n",
    "\n",
    "def last_archived_version():\n",
    "    '''\n",
    "    Gets the version number of the most recently archived model\n",
    "    '''\n",
    "    archive_folders = get_archive_folders()\n",
    "    \n",
    "    if len(archive_folders) == 0:\n",
    "        return None\n",
    "    version_nums = [int(re.search('\\d', fname).group()) for fname in archive_folders]\n",
    "    \n",
    "    return max(version_nums)\n",
    "\n",
    "# ============================= #\n",
    "# Getting / Saving Model Config #\n",
    "# ============================= #\n",
    "\n",
    "def save_model_config(d, model=None):\n",
    "    '''\n",
    "    Save the model configuration in the specified path \n",
    "    \n",
    "    NOTE\n",
    "    ----\n",
    "    - Assumes that if no model is passed, can find the best weights in the directory d and\n",
    "    load the model from that hdf5 file\n",
    "    '''\n",
    "    # Custom layers and metrics need to be passed in order to actually load the model\n",
    "    \n",
    "    # See if we have to load a model\n",
    "    if model is None:\n",
    "        best_model_path = best_weights(d)\n",
    "        with h5py.File(best_model_path, 'r') as f:\n",
    "            config_txt = json.dumps(json.loads(f.attrs['model_config'])['config'])\n",
    "    else:\n",
    "        config_txt = json.dumps(model.get_config())\n",
    "\n",
    "    fname = os.path.join(d, 'config.json')\n",
    "    \n",
    "    with open(fname, 'w') as f:\n",
    "        f.write(config_txt)\n",
    "        \n",
    "def archive_weights(weights_src):\n",
    "    '''\n",
    "    Moves the weights in the provided source to a new archive file and\n",
    "    stores txt files with model summary and configs\n",
    "    '''\n",
    "    # Get all the weigts in the weight_src directory\n",
    "    current_weights = all_weights_in_dir(weights_src)\n",
    "    \n",
    "    assert len(current_weights) !=0, \"No weights to move in the specified source directory\"\n",
    "    \n",
    "    # Most recent version\n",
    "    most_recent_version = last_archived_version()\n",
    "    \n",
    "    if most_recent_version is None:\n",
    "        new_vnum = 0\n",
    "    else:\n",
    "        new_vnum = most_recent_version + 1\n",
    "    \n",
    "    new_version_fname = 'models/weights_v{:d}'.format(new_vnum)\n",
    "    \n",
    "    # Double check this folder doesn't already exist\n",
    "    assert new_version_fname not in get_archive_folders(), \\\n",
    "    \"Archive folder path {} already exists\".format(new_version_fname)\n",
    "    \n",
    "    os.mkdir(new_version_fname)\n",
    "    \n",
    "    for wpath in current_weights:\n",
    "        weight_fname = os.path.split(wpath)[-1]\n",
    "        new_wpath = os.path.join(new_version_fname, weight_fname)\n",
    "        os.rename(wpath, new_wpath)  # effectively copies the weights\n",
    "        \n",
    "    # Check that there are no weights in the source directory\n",
    "    current_weights_after = all_weights_in_dir(weights_src)\n",
    "    assert len(current_weights_after) == 0, \\\n",
    "    \"Something went wrong, weights still remain: {}\".format(current_weights_after)\n",
    "    \n",
    "    # Save the model config to the new folder\n",
    "    save_model_config(new_version_fname)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Train testing model locally"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/10\n",
      " 40/288 [===>..........................] - ETA: 914s - loss: 0.6325 - accuracy: 0.8795 - true_pos: nan"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-25-b796c2c7ae4a>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m test_model.fit([token_enc_train, char_enc_train, word_end_ind_train, word_segment_mask_train], \n\u001b[0;32m----> 2\u001b[0;31m           np.expand_dims(tags_enc_train, 2), batch_size=2, callbacks=[ModelCheckpointBatch(MODEL_SAVE_PATH, period=2)])\n\u001b[0m",
      "\u001b[0;32m/Users/areiner/tensorflow/lib/python2.7/site-packages/keras/engine/training.pyc\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, x, y, batch_size, nb_epoch, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch)\u001b[0m\n\u001b[1;32m   1141\u001b[0m                               \u001b[0mval_f\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mval_f\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mval_ins\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mval_ins\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mshuffle\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mshuffle\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1142\u001b[0m                               \u001b[0mcallback_metrics\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcallback_metrics\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1143\u001b[0;31m                               initial_epoch=initial_epoch)\n\u001b[0m\u001b[1;32m   1144\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1145\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mevaluate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_size\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m32\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mverbose\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msample_weight\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/areiner/tensorflow/lib/python2.7/site-packages/keras/engine/training.pyc\u001b[0m in \u001b[0;36m_fit_loop\u001b[0;34m(self, f, ins, out_labels, batch_size, nb_epoch, verbose, callbacks, val_f, val_ins, shuffle, callback_metrics, initial_epoch)\u001b[0m\n\u001b[1;32m    841\u001b[0m                 \u001b[0mbatch_logs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'size'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch_ids\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    842\u001b[0m                 \u001b[0mcallbacks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mon_batch_begin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch_index\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_logs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 843\u001b[0;31m                 \u001b[0mouts\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mins_batch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    844\u001b[0m                 \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mouts\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlist\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    845\u001b[0m                     \u001b[0mouts\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mouts\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/areiner/tensorflow/lib/python2.7/site-packages/keras/backend/tensorflow_backend.pyc\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, inputs)\u001b[0m\n\u001b[1;32m   1601\u001b[0m         \u001b[0msession\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mget_session\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1602\u001b[0m         updated = session.run(self.outputs + [self.updates_op],\n\u001b[0;32m-> 1603\u001b[0;31m                               feed_dict=feed_dict)\n\u001b[0m\u001b[1;32m   1604\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mupdated\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moutputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1605\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/areiner/tensorflow/lib/python2.7/site-packages/tensorflow/python/client/session.pyc\u001b[0m in \u001b[0;36mrun\u001b[0;34m(self, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m    764\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    765\u001b[0m       result = self._run(None, fetches, feed_dict, options_ptr,\n\u001b[0;32m--> 766\u001b[0;31m                          run_metadata_ptr)\n\u001b[0m\u001b[1;32m    767\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    768\u001b[0m         \u001b[0mproto_data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf_session\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTF_GetBuffer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrun_metadata_ptr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/areiner/tensorflow/lib/python2.7/site-packages/tensorflow/python/client/session.pyc\u001b[0m in \u001b[0;36m_run\u001b[0;34m(self, handle, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m    962\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mfinal_fetches\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mfinal_targets\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    963\u001b[0m       results = self._do_run(handle, final_targets, final_fetches,\n\u001b[0;32m--> 964\u001b[0;31m                              feed_dict_string, options, run_metadata)\n\u001b[0m\u001b[1;32m    965\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    966\u001b[0m       \u001b[0mresults\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/areiner/tensorflow/lib/python2.7/site-packages/tensorflow/python/client/session.pyc\u001b[0m in \u001b[0;36m_do_run\u001b[0;34m(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m   1012\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mhandle\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1013\u001b[0m       return self._do_call(_run_fn, self._session, feed_dict, fetch_list,\n\u001b[0;32m-> 1014\u001b[0;31m                            target_list, options, run_metadata)\n\u001b[0m\u001b[1;32m   1015\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1016\u001b[0m       return self._do_call(_prun_fn, self._session, handle, feed_dict,\n",
      "\u001b[0;32m/Users/areiner/tensorflow/lib/python2.7/site-packages/tensorflow/python/client/session.pyc\u001b[0m in \u001b[0;36m_do_call\u001b[0;34m(self, fn, *args)\u001b[0m\n\u001b[1;32m   1019\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0m_do_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1020\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1021\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1022\u001b[0m     \u001b[0;32mexcept\u001b[0m \u001b[0merrors\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mOpError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1023\u001b[0m       \u001b[0mmessage\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcompat\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mas_text\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmessage\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/areiner/tensorflow/lib/python2.7/site-packages/tensorflow/python/client/session.pyc\u001b[0m in \u001b[0;36m_run_fn\u001b[0;34m(session, feed_dict, fetch_list, target_list, options, run_metadata)\u001b[0m\n\u001b[1;32m   1001\u001b[0m         return tf_session.TF_Run(session, options,\n\u001b[1;32m   1002\u001b[0m                                  \u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtarget_list\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1003\u001b[0;31m                                  status, run_metadata)\n\u001b[0m\u001b[1;32m   1004\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1005\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_prun_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msession\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhandle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "test_model.fit([token_enc_wg_train, char_enc_wg_train, word_end_ind_wg_train, word_segment_mask_wg_train], \n",
    "          np.expand_dims(tags_enc_train, 2), batch_size=2, callbacks=[ModelCheckpointBatch(MODEL_SAVE_PATH, period=2)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/2\n",
      " 24/288 [=>............................] - ETA: 724s - loss: 0.4661 - accuracy: 0.9617 - true_pos: nan"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-26-7308241686aa>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m test_model.fit_generator(data_generator(train, batch_size=2, nb_epoch=2, unk_neg_prob=0.05, unk_pos_prob=0.1), \n\u001b[1;32m      2\u001b[0m                          \u001b[0msamples_per_epoch\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtrain\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m                          nb_epoch=2, callbacks=[ModelCheckpointBatch(MODEL_SAVE_PATH, period=2)])\n\u001b[0m",
      "\u001b[0;32m/Users/areiner/tensorflow/lib/python2.7/site-packages/keras/engine/training.pyc\u001b[0m in \u001b[0;36mfit_generator\u001b[0;34m(self, generator, samples_per_epoch, nb_epoch, verbose, callbacks, validation_data, nb_val_samples, class_weight, max_q_size, nb_worker, pickle_safe, initial_epoch)\u001b[0m\n\u001b[1;32m   1506\u001b[0m                     outs = self.train_on_batch(x, y,\n\u001b[1;32m   1507\u001b[0m                                                \u001b[0msample_weight\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0msample_weight\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1508\u001b[0;31m                                                class_weight=class_weight)\n\u001b[0m\u001b[1;32m   1509\u001b[0m                 \u001b[0;32mexcept\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1510\u001b[0m                     \u001b[0m_stop\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mset\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/areiner/tensorflow/lib/python2.7/site-packages/keras/engine/training.pyc\u001b[0m in \u001b[0;36mtrain_on_batch\u001b[0;34m(self, x, y, sample_weight, class_weight)\u001b[0m\n\u001b[1;32m   1265\u001b[0m             \u001b[0mins\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mx\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0my\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0msample_weights\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1266\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_make_train_function\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1267\u001b[0;31m         \u001b[0moutputs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain_function\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mins\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1268\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moutputs\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1269\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0moutputs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/areiner/tensorflow/lib/python2.7/site-packages/keras/backend/tensorflow_backend.pyc\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, inputs)\u001b[0m\n\u001b[1;32m   1601\u001b[0m         \u001b[0msession\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mget_session\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1602\u001b[0m         updated = session.run(self.outputs + [self.updates_op],\n\u001b[0;32m-> 1603\u001b[0;31m                               feed_dict=feed_dict)\n\u001b[0m\u001b[1;32m   1604\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mupdated\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moutputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1605\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/areiner/tensorflow/lib/python2.7/site-packages/tensorflow/python/client/session.pyc\u001b[0m in \u001b[0;36mrun\u001b[0;34m(self, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m    764\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    765\u001b[0m       result = self._run(None, fetches, feed_dict, options_ptr,\n\u001b[0;32m--> 766\u001b[0;31m                          run_metadata_ptr)\n\u001b[0m\u001b[1;32m    767\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    768\u001b[0m         \u001b[0mproto_data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf_session\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTF_GetBuffer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrun_metadata_ptr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/areiner/tensorflow/lib/python2.7/site-packages/tensorflow/python/client/session.pyc\u001b[0m in \u001b[0;36m_run\u001b[0;34m(self, handle, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m    962\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mfinal_fetches\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mfinal_targets\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    963\u001b[0m       results = self._do_run(handle, final_targets, final_fetches,\n\u001b[0;32m--> 964\u001b[0;31m                              feed_dict_string, options, run_metadata)\n\u001b[0m\u001b[1;32m    965\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    966\u001b[0m       \u001b[0mresults\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/areiner/tensorflow/lib/python2.7/site-packages/tensorflow/python/client/session.pyc\u001b[0m in \u001b[0;36m_do_run\u001b[0;34m(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m   1012\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mhandle\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1013\u001b[0m       return self._do_call(_run_fn, self._session, feed_dict, fetch_list,\n\u001b[0;32m-> 1014\u001b[0;31m                            target_list, options, run_metadata)\n\u001b[0m\u001b[1;32m   1015\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1016\u001b[0m       return self._do_call(_prun_fn, self._session, handle, feed_dict,\n",
      "\u001b[0;32m/Users/areiner/tensorflow/lib/python2.7/site-packages/tensorflow/python/client/session.pyc\u001b[0m in \u001b[0;36m_do_call\u001b[0;34m(self, fn, *args)\u001b[0m\n\u001b[1;32m   1019\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0m_do_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1020\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1021\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1022\u001b[0m     \u001b[0;32mexcept\u001b[0m \u001b[0merrors\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mOpError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1023\u001b[0m       \u001b[0mmessage\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcompat\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mas_text\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmessage\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/areiner/tensorflow/lib/python2.7/site-packages/tensorflow/python/client/session.pyc\u001b[0m in \u001b[0;36m_run_fn\u001b[0;34m(session, feed_dict, fetch_list, target_list, options, run_metadata)\u001b[0m\n\u001b[1;32m   1001\u001b[0m         return tf_session.TF_Run(session, options,\n\u001b[1;32m   1002\u001b[0m                                  \u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtarget_list\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1003\u001b[0;31m                                  status, run_metadata)\n\u001b[0m\u001b[1;32m   1004\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1005\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_prun_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msession\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhandle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "test_model.fit_generator(data_generator(train, batch_size=2, nb_epoch=2, unk_neg_prob=0.05, unk_pos_prob=0.1), \n",
    "                         samples_per_epoch=train.shape[0], \n",
    "                         nb_epoch=2, callbacks=[ModelCheckpointBatch(MODEL_SAVE_PATH, period=2)])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Train full model in cloud"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 539,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Loading the model\n",
    "BEST_WEIGHTS_PATH = best_weights(MODEL_WEIGHTS_DIR)\n",
    "# model.load_weights(BEST_WEIGHTS_PATH)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/10\n",
      "288/288 [==============================] - 745s - loss: 0.2146 - accuracy: 0.9600 - true_pos: nan   \n",
      "Epoch 2/10\n",
      "102/288 [=========>....................] - ETA: 482s - loss: 0.1390 - accuracy: 0.9646 - true_pos: nan"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-63-0f97f9d5f666>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m model.fit([token_enc_train, char_enc_train, word_end_ind_train, word_segment_mask_train], \n\u001b[0;32m----> 2\u001b[0;31m           np.expand_dims(tags_enc_train, 2), batch_size=2)\n\u001b[0m",
      "\u001b[0;32m/Users/areiner/tensorflow/lib/python2.7/site-packages/keras/engine/training.pyc\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, x, y, batch_size, nb_epoch, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch)\u001b[0m\n\u001b[1;32m   1141\u001b[0m                               \u001b[0mval_f\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mval_f\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mval_ins\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mval_ins\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mshuffle\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mshuffle\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1142\u001b[0m                               \u001b[0mcallback_metrics\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcallback_metrics\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1143\u001b[0;31m                               initial_epoch=initial_epoch)\n\u001b[0m\u001b[1;32m   1144\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1145\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mevaluate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_size\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m32\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mverbose\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msample_weight\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/areiner/tensorflow/lib/python2.7/site-packages/keras/engine/training.pyc\u001b[0m in \u001b[0;36m_fit_loop\u001b[0;34m(self, f, ins, out_labels, batch_size, nb_epoch, verbose, callbacks, val_f, val_ins, shuffle, callback_metrics, initial_epoch)\u001b[0m\n\u001b[1;32m    841\u001b[0m                 \u001b[0mbatch_logs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'size'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch_ids\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    842\u001b[0m                 \u001b[0mcallbacks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mon_batch_begin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch_index\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_logs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 843\u001b[0;31m                 \u001b[0mouts\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mins_batch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    844\u001b[0m                 \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mouts\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlist\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    845\u001b[0m                     \u001b[0mouts\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mouts\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/areiner/tensorflow/lib/python2.7/site-packages/keras/backend/tensorflow_backend.pyc\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, inputs)\u001b[0m\n\u001b[1;32m   1601\u001b[0m         \u001b[0msession\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mget_session\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1602\u001b[0m         updated = session.run(self.outputs + [self.updates_op],\n\u001b[0;32m-> 1603\u001b[0;31m                               feed_dict=feed_dict)\n\u001b[0m\u001b[1;32m   1604\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mupdated\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moutputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1605\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/areiner/tensorflow/lib/python2.7/site-packages/tensorflow/python/client/session.pyc\u001b[0m in \u001b[0;36mrun\u001b[0;34m(self, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m    764\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    765\u001b[0m       result = self._run(None, fetches, feed_dict, options_ptr,\n\u001b[0;32m--> 766\u001b[0;31m                          run_metadata_ptr)\n\u001b[0m\u001b[1;32m    767\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    768\u001b[0m         \u001b[0mproto_data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf_session\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTF_GetBuffer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrun_metadata_ptr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/areiner/tensorflow/lib/python2.7/site-packages/tensorflow/python/client/session.pyc\u001b[0m in \u001b[0;36m_run\u001b[0;34m(self, handle, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m    962\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mfinal_fetches\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mfinal_targets\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    963\u001b[0m       results = self._do_run(handle, final_targets, final_fetches,\n\u001b[0;32m--> 964\u001b[0;31m                              feed_dict_string, options, run_metadata)\n\u001b[0m\u001b[1;32m    965\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    966\u001b[0m       \u001b[0mresults\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/areiner/tensorflow/lib/python2.7/site-packages/tensorflow/python/client/session.pyc\u001b[0m in \u001b[0;36m_do_run\u001b[0;34m(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m   1012\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mhandle\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1013\u001b[0m       return self._do_call(_run_fn, self._session, feed_dict, fetch_list,\n\u001b[0;32m-> 1014\u001b[0;31m                            target_list, options, run_metadata)\n\u001b[0m\u001b[1;32m   1015\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1016\u001b[0m       return self._do_call(_prun_fn, self._session, handle, feed_dict,\n",
      "\u001b[0;32m/Users/areiner/tensorflow/lib/python2.7/site-packages/tensorflow/python/client/session.pyc\u001b[0m in \u001b[0;36m_do_call\u001b[0;34m(self, fn, *args)\u001b[0m\n\u001b[1;32m   1019\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0m_do_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1020\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1021\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1022\u001b[0m     \u001b[0;32mexcept\u001b[0m \u001b[0merrors\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mOpError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1023\u001b[0m       \u001b[0mmessage\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcompat\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mas_text\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmessage\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/areiner/tensorflow/lib/python2.7/site-packages/tensorflow/python/client/session.pyc\u001b[0m in \u001b[0;36m_run_fn\u001b[0;34m(session, feed_dict, fetch_list, target_list, options, run_metadata)\u001b[0m\n\u001b[1;32m   1001\u001b[0m         return tf_session.TF_Run(session, options,\n\u001b[1;32m   1002\u001b[0m                                  \u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtarget_list\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1003\u001b[0;31m                                  status, run_metadata)\n\u001b[0m\u001b[1;32m   1004\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1005\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_prun_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msession\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhandle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "model.fit([token_enc_train, char_enc_train, word_end_ind_train, word_segment_mask_train], \n",
    "          np.expand_dims(tags_enc_train, 2), batch_size=100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'model' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-23-e467dda2cd9d>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m model.fit_generator(data_generator(wikiner, batch_size=100, nb_epoch=10, unk_neg_prob=0.05, unk_pos_prob=0.1), \n\u001b[0m\u001b[1;32m      2\u001b[0m                     \u001b[0msamples_per_epoch\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mwikiner\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m                     nb_epoch=10, callbacks=[ModelCheckpointBatch(MODEL_SAVE_PATH, period=100)])\n",
      "\u001b[0;31mNameError\u001b[0m: name 'model' is not defined"
     ]
    }
   ],
   "source": [
    "model.fit_generator(data_generator(wikiner, batch_size=100, nb_epoch=10, unk_neg_prob=0.05, unk_pos_prob=0.1), \n",
    "                    samples_per_epoch=wikiner.shape[0], \n",
    "                    nb_epoch=10, callbacks=[ModelCheckpointBatch(MODEL_SAVE_PATH, period=100)])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# VII. Model Validation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "np.set_printoptions(precision=4, suppress=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## a. Methods for validation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Masks**\n",
    "- Tokens -> 0\n",
    "- Characters -> 0\n",
    "- Tags -> -1\n",
    "- Word Indices -> -1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 586,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# =================================================== #\n",
    "# Mask true tokens to their spots in the full strings #\n",
    "# =================================================== #\n",
    "\n",
    "def predict_char_tags(word_start_inds, word_end_inds, predicted_tag_enc):\n",
    "    '''\n",
    "    Returns lists the same length as the number of characters in the document string\n",
    "    that encodes characters as 1 when they belong to a tagged entity\n",
    "    \n",
    "    Paramters\n",
    "    =========\n",
    "    word_start_inds : numpy array\n",
    "        The padded word start indices for a single example\n",
    "        \n",
    "    word_end_inds : numpy array\n",
    "        The padded word end indices for a single example\n",
    "    \n",
    "    predicted_tag_enc : numpy array\n",
    "        The predicted token-wise tag encoding for a single example with padded (nonsense) predictions left in\n",
    "        Note: this is the form of the output from the model\n",
    "    '''\n",
    "    # 1. Get the length of the document string (num of characters in example) from the word indices\n",
    "    filt = word_end_inds!=-1\n",
    "    word_start_inds = word_start_inds[filt]\n",
    "    word_end_inds = word_end_inds[filt]\n",
    "    doc_string_len = word_end_inds.max() + 1\n",
    "    \n",
    "    # 2. Remove the padded tags\n",
    "    predicted_tag_enc = predicted_tag_enc[filt]\n",
    "\n",
    "    # 3. Fill in a numpy array of tags the length of the unpadded document string\n",
    "    char_tags = np.zeros(doc_string_len, dtype='int64')\n",
    "    \n",
    "    word_start_ind_tag = word_start_inds[predicted_tag_enc==1]\n",
    "    word_end_ind_tag = word_end_inds[predicted_tag_enc==1]\n",
    "    word_span_tag = zip(word_start_ind_tag, word_end_ind_tag)\n",
    "    \n",
    "    for (s, e) in word_span_tag:\n",
    "        char_tags[s:e+1] = 1\n",
    "    \n",
    "    return list(char_tags)\n",
    "    \n",
    "def tagged_entities_in_docstring(char_lists,\n",
    "                                 word_start_inds_arr, word_end_inds_arr,\n",
    "                                 yhats,\n",
    "                                 true_char_tag_enc_lists,\n",
    "                                 thresh=0.5):\n",
    "    '''\n",
    "    Takes the model output and returns:\n",
    "    (1) the array of characters\n",
    "    (2) the characters that actually belong to tags\n",
    "    (3) the characters we estimate belong to tags\n",
    "    \n",
    "    Parameters\n",
    "    ==========\n",
    "    char_lists : nested list\n",
    "        nested list of character lists for the examples passed to produce the provided\n",
    "        yhat model output\n",
    "        \n",
    "    word_star_inds_arr : numpy array\n",
    "        padded array of word start indices\n",
    "    \n",
    "    word_end_inds_arr : numpy array\n",
    "        padded array of word end indices\n",
    "        \n",
    "    yhats : numpy array\n",
    "        output of the model\n",
    "        \n",
    "    true_char_tag_enc_lists : nested list\n",
    "        true char by char tag encoding of the model\n",
    "        \n",
    "    Returns\n",
    "    =======\n",
    "    res : list of 3-tuple\n",
    "        each tuple corresponds to a single training example / document / partial document\n",
    "        and is composed of\n",
    "        (i) sequence of characters that make up the document \n",
    "        (ii) sequence of predicted tags for each character representing a mask of what tokens were tagged\n",
    "        (iii) sequence of true tags for each character representing a mask of that tokens were tagged\n",
    "    \n",
    "    NOTE\n",
    "    ----\n",
    "    All the inputs should share the same size in the 0th dimension.\n",
    "    For dim 1, the sizes should be:\n",
    "        (len of doc i in characters, \n",
    "        max document len in tokens,\n",
    "        max document len in tokens,\n",
    "        max document len in tokens, \n",
    "        len of doc i in characters)\n",
    "    '''\n",
    "    res = []\n",
    "    for i in range(len(word_start_inds_arr)):\n",
    "        \n",
    "        # 1. Get each example passed\n",
    "        \n",
    "        # Get the predictions\n",
    "        char_list = char_lists[i]\n",
    "        word_start_inds = word_start_inds_arr[i]\n",
    "        word_end_inds = word_end_inds_arr[i]\n",
    "        yhat = yhats[i]\n",
    "        predicted_tag_enc =(yhat >= thresh).astype('int64')\n",
    "        \n",
    "        # Extract the true character tags\n",
    "        true_char_tag_enc = true_char_tag_enc_lists[i]\n",
    "        \n",
    "        # Calculate the predicted char_tags\n",
    "        predicted_char_tags = predict_char_tags(word_start_inds, word_end_inds, predicted_tag_enc)\n",
    "        \n",
    "        # Append the example results\n",
    "        res.append((np.array(char_list), np.array(predicted_char_tags), np.array(true_char_tag_enc)))\n",
    "    \n",
    "    return res\n",
    "\n",
    "def add_tagged_char_mask_to_df(model, df, batch_size=5, thresh=0.5):\n",
    "    \n",
    "    token_enc, char_enc, tags_enc, word_start_ind, word_end_ind, word_segment_mask = generate_padded_data(df)\n",
    "    x = [token_enc, char_enc, word_end_ind, word_segment_mask]\n",
    "    if 'prob' in df.columns:\n",
    "        y_preds = df.prob.tolist()\n",
    "    else:\n",
    "        y_preds = model.predict(x, batch_size=batch_size)[..., 0]\n",
    "        \n",
    "    char_lists = df.chars.tolist()\n",
    "    true_char_tag_enc_lists = df.char_tag_mask.tolist()\n",
    "    \n",
    "    res_list = tagged_entities_in_docstring(char_lists,\n",
    "                                                 word_start_ind, word_end_ind, \n",
    "                                                 y_preds,\n",
    "                                                 true_char_tag_enc_lists, \n",
    "                                                 thresh=thresh)\n",
    "    pred_char_tag = [r[1] for r in res_list]\n",
    "    df['pred_char_tag_mask'] = pred_char_tag\n",
    "    return df\n",
    "\n",
    "# ================================ #\n",
    "# Add Predictions to the Dataframe #\n",
    "# ================================ #\n",
    "\n",
    "def add_predictions_to_df(x, model, df, batch_size=5, thresh=0.5):\n",
    "    if 'prob' in df.columns:\n",
    "        y_preds = df.prob.tolist()\n",
    "    else:\n",
    "        y_preds = model.predict(x, batch_size=batch_size)[..., 0]\n",
    "    y_preds_nopad = []\n",
    "    yhats = []\n",
    "    for i in range(len(y_preds)):\n",
    "        token_mask = x[0][i] != 0\n",
    "        y_pred_nopad = y_preds[i][token_mask]\n",
    "        y_preds_nopad.append(y_pred_nopad)\n",
    "        yhats.append((y_pred_nopad > thresh).astype('int32'))\n",
    "    df['prob'] = y_preds_nopad\n",
    "    df['yhat'] = yhats\n",
    "    return df\n",
    "\n",
    "def calculate_and_add_preds_to_df(model, df, batch_size=5, thresh=0.5):\n",
    "    '''\n",
    "    Calculate and add preds\n",
    "    '''\n",
    "    token_enc, char_enc, tags_enc, word_start_ind, word_end_ind, word_segment_mask = generate_padded_data(df)\n",
    "    x = [token_enc, char_enc, word_end_ind, word_segment_mask]\n",
    "    df = add_predictions_to_df(x, model, df, batch_size=batch_size)\n",
    "    return df\n",
    "\n",
    "# ==================== #\n",
    "# Metrics Calculations #\n",
    "# ==================== #\n",
    "\n",
    "def concat_col(col):\n",
    "    '''\n",
    "    Concatentates a column of lists into a single numpy array\n",
    "    '''\n",
    "    res = []\n",
    "\n",
    "    for el in col:\n",
    "        res.extend(el)\n",
    "\n",
    "    return np.array(res)\n",
    "\n",
    "def get_probs(df):\n",
    "    '''\n",
    "    After we add probabilities, get a flattened array of all the \n",
    "    predicted probabilities\n",
    "    '''\n",
    "    return concat_col(df.probs)\n",
    "\n",
    "def get_tags_enc(df):\n",
    "    '''\n",
    "    Return the encoded tags from the dataframe in a flattened array\n",
    "    '''\n",
    "    return concat_col(df.tags_enc)\n",
    "\n",
    "def compute_precision(yhat, ytrue):\n",
    "    '''\n",
    "    Compute the precision\n",
    "    '''\n",
    "    num_true_pos, num_flagged = calc_true_pos_num_flagged(yhat, ytrue)\n",
    "    \n",
    "    return float(num_true_pos) / num_flagged\n",
    "    \n",
    "def calc_true_pos_num_flagged(yhat, ytrue):\n",
    "    num_true_pos = ((yhat==1) & (yhat==ytrue)).sum()\n",
    "    num_flagged = (yhat==1).sum()\n",
    "    \n",
    "    return num_true_pos, num_flagged\n",
    "\n",
    "def calc_yhat(probs, thresh=0.5):\n",
    "    return (probs > thresh).astype('int')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## c. Run Predictions on the train and test dataframes and store them"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Examples to Explore"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 588,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df = wikigold\n",
    "token_enc, char_enc, tags_enc, word_start_ind, word_end_ind, word_segment_mask = generate_padded_data(df)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Run predictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-71-6f3ca3816be2>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mtrain_p\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcalculate_and_add_preds_to_df\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrain\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m<ipython-input-69-c2b5c7c58178>\u001b[0m in \u001b[0;36mcalculate_and_add_preds_to_df\u001b[0;34m(model, df, batch_size, thresh)\u001b[0m\n\u001b[1;32m    151\u001b[0m     \u001b[0mtoken_enc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mchar_enc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtags_enc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mword_start_ind\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mword_end_ind\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mword_segment_mask\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mgenerate_padded_data\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdf\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    152\u001b[0m     \u001b[0mx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mtoken_enc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mchar_enc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mword_end_ind\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mword_segment_mask\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 153\u001b[0;31m     \u001b[0mdf\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0madd_predictions_to_df\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_size\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mbatch_size\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    154\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0mdf\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-69-c2b5c7c58178>\u001b[0m in \u001b[0;36madd_predictions_to_df\u001b[0;34m(x, model, df, batch_size, thresh)\u001b[0m\n\u001b[1;32m    133\u001b[0m         \u001b[0my_preds\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mprob\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtolist\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    134\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 135\u001b[0;31m         \u001b[0my_preds\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpredict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_size\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mbatch_size\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    136\u001b[0m     \u001b[0my_preds_nopad\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    137\u001b[0m     \u001b[0myhats\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/areiner/tensorflow/lib/python2.7/site-packages/keras/engine/training.pyc\u001b[0m in \u001b[0;36mpredict\u001b[0;34m(self, x, batch_size, verbose)\u001b[0m\n\u001b[1;32m   1217\u001b[0m         \u001b[0mf\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpredict_function\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1218\u001b[0m         return self._predict_loop(f, ins,\n\u001b[0;32m-> 1219\u001b[0;31m                                   batch_size=batch_size, verbose=verbose)\n\u001b[0m\u001b[1;32m   1220\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1221\u001b[0m     def train_on_batch(self, x, y,\n",
      "\u001b[0;32m/Users/areiner/tensorflow/lib/python2.7/site-packages/keras/engine/training.pyc\u001b[0m in \u001b[0;36m_predict_loop\u001b[0;34m(self, f, ins, batch_size, verbose)\u001b[0m\n\u001b[1;32m    895\u001b[0m                 \u001b[0mins_batch\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mslice_X\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mins\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_ids\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    896\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 897\u001b[0;31m             \u001b[0mbatch_outs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mins_batch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    898\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch_outs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlist\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    899\u001b[0m                 \u001b[0mbatch_outs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mbatch_outs\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/areiner/tensorflow/lib/python2.7/site-packages/keras/backend/tensorflow_backend.pyc\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, inputs)\u001b[0m\n\u001b[1;32m   1601\u001b[0m         \u001b[0msession\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mget_session\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1602\u001b[0m         updated = session.run(self.outputs + [self.updates_op],\n\u001b[0;32m-> 1603\u001b[0;31m                               feed_dict=feed_dict)\n\u001b[0m\u001b[1;32m   1604\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mupdated\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moutputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1605\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/areiner/tensorflow/lib/python2.7/site-packages/tensorflow/python/client/session.pyc\u001b[0m in \u001b[0;36mrun\u001b[0;34m(self, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m    764\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    765\u001b[0m       result = self._run(None, fetches, feed_dict, options_ptr,\n\u001b[0;32m--> 766\u001b[0;31m                          run_metadata_ptr)\n\u001b[0m\u001b[1;32m    767\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    768\u001b[0m         \u001b[0mproto_data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf_session\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTF_GetBuffer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrun_metadata_ptr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/areiner/tensorflow/lib/python2.7/site-packages/tensorflow/python/client/session.pyc\u001b[0m in \u001b[0;36m_run\u001b[0;34m(self, handle, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m    962\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mfinal_fetches\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mfinal_targets\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    963\u001b[0m       results = self._do_run(handle, final_targets, final_fetches,\n\u001b[0;32m--> 964\u001b[0;31m                              feed_dict_string, options, run_metadata)\n\u001b[0m\u001b[1;32m    965\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    966\u001b[0m       \u001b[0mresults\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/areiner/tensorflow/lib/python2.7/site-packages/tensorflow/python/client/session.pyc\u001b[0m in \u001b[0;36m_do_run\u001b[0;34m(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m   1012\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mhandle\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1013\u001b[0m       return self._do_call(_run_fn, self._session, feed_dict, fetch_list,\n\u001b[0;32m-> 1014\u001b[0;31m                            target_list, options, run_metadata)\n\u001b[0m\u001b[1;32m   1015\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1016\u001b[0m       return self._do_call(_prun_fn, self._session, handle, feed_dict,\n",
      "\u001b[0;32m/Users/areiner/tensorflow/lib/python2.7/site-packages/tensorflow/python/client/session.pyc\u001b[0m in \u001b[0;36m_do_call\u001b[0;34m(self, fn, *args)\u001b[0m\n\u001b[1;32m   1019\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0m_do_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1020\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1021\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1022\u001b[0m     \u001b[0;32mexcept\u001b[0m \u001b[0merrors\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mOpError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1023\u001b[0m       \u001b[0mmessage\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcompat\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mas_text\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmessage\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/areiner/tensorflow/lib/python2.7/site-packages/tensorflow/python/client/session.pyc\u001b[0m in \u001b[0;36m_run_fn\u001b[0;34m(session, feed_dict, fetch_list, target_list, options, run_metadata)\u001b[0m\n\u001b[1;32m   1001\u001b[0m         return tf_session.TF_Run(session, options,\n\u001b[1;32m   1002\u001b[0m                                  \u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtarget_list\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1003\u001b[0;31m                                  status, run_metadata)\n\u001b[0m\u001b[1;32m   1004\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1005\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_prun_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msession\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhandle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "df = calculate_and_add_preds_to_df(model, df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Get flattened predictions and labels from the dataframe \n",
    "probs = get_probs(df)\n",
    "yhat = calc_yhat(probs, thresh=0.5)\n",
    "ytrue = get_tags_enc(df)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Compute Metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 596,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.metrics import classification_report, precision_recall_curve"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 597,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'yhat' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-597-1e8fdd350b8c>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0;32mprint\u001b[0m \u001b[0mclassification_report\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mytrue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0myhat\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m: name 'yhat' is not defined"
     ]
    }
   ],
   "source": [
    "print classification_report(ytrue, yhat)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "precision, recall, thresholds = precision_recall_curve(ytrue, probs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Explore Examples"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 589,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'model' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-589-08a5b3564b43>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0mx_in\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mtoken_enc\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mchar_enc\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mword_end_ind\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mword_segment_mask\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 4\u001b[0;31m \u001b[0mout\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpredict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx_in\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m:\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      5\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      6\u001b[0m \u001b[0mtoken_ex\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0marray\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdocument\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'model' is not defined"
     ]
    }
   ],
   "source": [
    "i = 0\n",
    "x_in = [token_enc[i:i+1], char_enc[i:i+1], word_end_ind[i:i+1], word_segment_mask[i:i+1]]\n",
    "\n",
    "out = model.predict(x_in)[0, :, 0]\n",
    "\n",
    "token_ex = np.array(df.document[i])\n",
    "tags_ex = np.array(df.tags[i])\n",
    "tags_enc_ex = np.array(df.tags_enc[i])\n",
    "\n",
    "m = token_ex[(tags_enc_ex.nonzero()[0])]\n",
    "\n",
    "print m\n",
    "print 'Real tags:', zip(m, out[tags_enc_ex.nonzero()[0]])\n",
    "out_sorted = out.argsort()\n",
    "out_sorted = out_sorted[out_sorted<len(token_ex)]\n",
    "print \"Lowest scores:\", token_ex[out_sorted[:30]]\n",
    "print \"Highest scores:\", token_ex[out_sorted[-20:]]\n",
    "print \"Highest scores:\", out[out_sorted[-20:]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Old Stuff"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Q: How can we determine the end of word indices from the tokens?\n",
    "- we need to use the provided tokens and reconstruct the original string\n",
    "    - We do this via an \"untokenize\" function found off the shelf online\n",
    "    - If we tokenize again (using nltk word_tokenize) do we get the same result?   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 312,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "``\n",
      "\"\n",
      "bonus tracks: \" Battle of\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'bonus tracks:\" Battle of One\"( an original song that was also set'"
      ]
     },
     "execution_count": 312,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print word_tokenize(res.iloc[4].document_string)[257]\n",
    "print res.iloc[4].document[257]\n",
    "print untokenize(res.iloc[4].document[254:260])\n",
    "\n",
    "def untokenize2(tokens):\n",
    "    import string\n",
    "    return \"\".join([\" \"+i if not i.startswith(\"'\") and i not in string.punctuation else i for i in tokens]).strip()\n",
    "untokenize2(res.iloc[4].document[254:270])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Q: Why is the true positive metric for the keras model messing up and returning nan?\n",
    "- Extracting the variables from the tensorflow model and constructing the accuracy metric outside works..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "y_true = tf.placeholder(dtype='float32')\n",
    "y_pred = model.output\n",
    "\n",
    "def true_pos(y_true, y_pred):\n",
    "    den = tf.reduce_sum(tf.cast(tf.equal(tf.round(y_pred), 1) & tf.not_equal(y_true, -1), dtype='float32'))\n",
    "\n",
    "    i = tf.equal(y_true, tf.round(y_pred)) & \\\n",
    "            tf.equal(1., tf.round(y_pred)) & \\\n",
    "            tf.not_equal(y_true, -1)\n",
    "\n",
    "    num = tf.reduce_sum(tf.cast(i, dtype='float32'))\n",
    "\n",
    "    frac = tf.select(den==0., 0., num / den)\n",
    "\n",
    "    return frac\n",
    "\n",
    "frac = true_pos(y_true, y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Generate the data\n",
    "token_enc_train, char_enc_train, tags_enc_train, word_start_ind_train, word_end_ind_train = \\\n",
    "                                                                                        generate_padded_data(train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sess = tf.InteractiveSession()\n",
    "tf.global_variables_initializer().run()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "fd = dict(zip(model.input, [token_enc_train[:2,:,],\n",
    "                           char_enc_train[:2, :],\n",
    "                           word_end_ind_train[:2, :]]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.0 0.0 26.0\n",
      "0.0 0.0 3.0\n",
      "0.025641 1.0 39.0\n",
      "0.0 0.0 40.0\n",
      "0.047619 1.0 21.0\n",
      "0.0952381 2.0 21.0\n",
      "0.0 0.0 44.0\n",
      "0.0 0.0 42.0\n",
      "0.0 0.0 43.0\n",
      "0.0 0.0 35.0\n",
      "0.037037 2.0 54.0\n",
      "0.0714286 2.0 28.0\n",
      "0.08 2.0 25.0\n",
      "0.03125 1.0 32.0\n",
      "0.0655738 4.0 61.0\n",
      "0.0434783 1.0 23.0\n",
      "0.0714286 2.0 28.0\n",
      "0.0 0.0 27.0\n",
      "0.171429 6.0 35.0\n",
      "0.04 2.0 50.0\n",
      "0.0 0.0 41.0\n",
      "0.030303 1.0 33.0\n",
      "0.0 0.0 48.0\n",
      "0.0 0.0 46.0\n",
      "0.0 0.0 30.0\n",
      "0.0 0.0 33.0\n",
      "0.0 0.0 42.0\n",
      "0.0 0.0 50.0\n",
      "0.0526316 2.0 38.0\n",
      "0.0 0.0 53.0\n",
      "0.0689655 2.0 29.0\n",
      "0.0 0.0 14.0\n",
      "0.0 0.0 53.0\n",
      "0.0 0.0 34.0\n",
      "0.0 0.0 28.0\n",
      "0.0 0.0 14.0\n",
      "0.0 0.0 9.0\n",
      "0.0 0.0 73.0\n",
      "0.0 0.0 68.0\n",
      "0.0454545 4.0 88.0\n",
      "0.0833333 3.0 36.0\n",
      "0.0714286 2.0 28.0\n",
      "0.0 0.0 39.0\n",
      "0.0 0.0 27.0\n",
      "0.0 0.0 31.0\n",
      "0.097561 4.0 41.0\n",
      "0.215686 11.0 51.0\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-132-6e1569e3e94c>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      3\u001b[0m                                                                      \u001b[0mchar_enc_train\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m                                                                      \u001b[0mword_end_ind_train\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 5\u001b[0;31m                                                                      np.expand_dims(tags_enc_train[i:i+1], 2)]\n\u001b[0m\u001b[1;32m      6\u001b[0m                                          )\n\u001b[1;32m      7\u001b[0m                                      )\n",
      "\u001b[0;32m/Users/areiner/tensorflow/lib/python2.7/site-packages/tensorflow/python/client/session.pyc\u001b[0m in \u001b[0;36mrun\u001b[0;34m(self, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m    764\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    765\u001b[0m       result = self._run(None, fetches, feed_dict, options_ptr,\n\u001b[0;32m--> 766\u001b[0;31m                          run_metadata_ptr)\n\u001b[0m\u001b[1;32m    767\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    768\u001b[0m         \u001b[0mproto_data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf_session\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTF_GetBuffer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrun_metadata_ptr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/areiner/tensorflow/lib/python2.7/site-packages/tensorflow/python/client/session.pyc\u001b[0m in \u001b[0;36m_run\u001b[0;34m(self, handle, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m    962\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mfinal_fetches\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mfinal_targets\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    963\u001b[0m       results = self._do_run(handle, final_targets, final_fetches,\n\u001b[0;32m--> 964\u001b[0;31m                              feed_dict_string, options, run_metadata)\n\u001b[0m\u001b[1;32m    965\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    966\u001b[0m       \u001b[0mresults\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/areiner/tensorflow/lib/python2.7/site-packages/tensorflow/python/client/session.pyc\u001b[0m in \u001b[0;36m_do_run\u001b[0;34m(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m   1012\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mhandle\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1013\u001b[0m       return self._do_call(_run_fn, self._session, feed_dict, fetch_list,\n\u001b[0;32m-> 1014\u001b[0;31m                            target_list, options, run_metadata)\n\u001b[0m\u001b[1;32m   1015\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1016\u001b[0m       return self._do_call(_prun_fn, self._session, handle, feed_dict,\n",
      "\u001b[0;32m/Users/areiner/tensorflow/lib/python2.7/site-packages/tensorflow/python/client/session.pyc\u001b[0m in \u001b[0;36m_do_call\u001b[0;34m(self, fn, *args)\u001b[0m\n\u001b[1;32m   1019\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0m_do_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1020\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1021\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1022\u001b[0m     \u001b[0;32mexcept\u001b[0m \u001b[0merrors\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mOpError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1023\u001b[0m       \u001b[0mmessage\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcompat\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mas_text\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmessage\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/areiner/tensorflow/lib/python2.7/site-packages/tensorflow/python/client/session.pyc\u001b[0m in \u001b[0;36m_run_fn\u001b[0;34m(session, feed_dict, fetch_list, target_list, options, run_metadata)\u001b[0m\n\u001b[1;32m   1001\u001b[0m         return tf_session.TF_Run(session, options,\n\u001b[1;32m   1002\u001b[0m                                  \u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtarget_list\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1003\u001b[0;31m                                  status, run_metadata)\n\u001b[0m\u001b[1;32m   1004\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1005\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_prun_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msession\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhandle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "for i in range(100):\n",
    "    f, n, d = sess.run([frac, num, den], feed_dict=dict(zip(model.input + [y_true], [token_enc_train[i:i+1], \n",
    "                                                                     char_enc_train[i:i+1],\n",
    "                                                                     word_end_ind_train[i:i+1],\n",
    "                                                                     np.expand_dims(tags_enc_train[i:i+1], 2)]\n",
    "                                         )\n",
    "                                     )\n",
    "            )\n",
    "    \n",
    "\n",
    "    print f, n, d"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Old Segmentation Layer that didn't account for padding properly"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "class SegmentLayerOld(Layer):\n",
    "    '''\n",
    "    Takes a segmented sum\n",
    "    '''\n",
    "    def __init__(self, seg_func_name='sum', **kwargs):\n",
    "        super(SegmentLayer, self).__init__(**kwargs)\n",
    "        if seg_func_name == 'sum':\n",
    "            self.seg_func = tf.segment_sum\n",
    "        elif seg_func_name == 'mean':\n",
    "            self.seg_func = tf.segment_mean\n",
    "        elif seg_func_name == 'max':\n",
    "            self.seg_func = tf.segment_max\n",
    "        else:\n",
    "            self.seg_func = tf.segment_sum\n",
    "        \n",
    "    def build(self, input_shape):\n",
    "        super(SegmentLayer, self).build(input_shape)\n",
    "    \n",
    "    def compute_mask(self, x, mask=None):\n",
    "        return K.cast(K.not_equal(x[1], -1), 'bool')\n",
    "    \n",
    "    def call(self, x, mask=None):\n",
    "        rnn_inp = x[0]\n",
    "        segment_mask = x[2]\n",
    "\n",
    "        def f(inp):\n",
    "            '''\n",
    "            Performs a segmented sum on each input of (max_char_len, char_lstm_dim)\n",
    "            '''\n",
    "            mat = inp[0]\n",
    "            seg_mask = inp[1]\n",
    "            seg_sum = self.seg_func(mat, seg_mask)\n",
    "            return seg_sum\n",
    "        \n",
    "        map_fn_out = tf.map_fn(f, elems=(rnn_inp, segment_mask), dtype='float32')\n",
    "        \n",
    "        return map_fn_out\n",
    "    \n",
    "    def get_output_shape_for(self, input_shape):\n",
    "        rnn_shape = input_shape[0]\n",
    "        ind_shape = input_shape[1]\n",
    "        \n",
    "        return (rnn_shape[0], ind_shape[1], rnn_shape[2])"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "tensorflow_kernel",
   "language": "python",
   "name": "tensorflow_kernel"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}