6_lstm.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "8tQJd2YSCfWR"
   },
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "D7tqLMoKF6uq"
   },
   "source": [
    "Deep Learning\n",
    "=============\n",
    "\n",
    "Assignment 6\n",
    "------------\n",
    "\n",
    "After training a skip-gram model in `5_word2vec.ipynb`, the goal of this notebook is to train a LSTM character model over [Text8](http://mattmahoney.net/dc/textdata) data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "cellView": "both",
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "MvEblsgEXxrd"
   },
   "outputs": [],
   "source": [
    "# These are all the modules we'll be using later. Make sure you can import them\n",
    "# before proceeding further.\n",
    "import os\n",
    "import numpy as np\n",
    "import random\n",
    "import string\n",
    "import tensorflow as tf\n",
    "import zipfile\n",
    "from six.moves import range\n",
    "from six.moves.urllib.request import urlretrieve"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "cellView": "both",
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "output_extras": [
      {
       "item_id": 1
      }
     ]
    },
    "colab_type": "code",
    "collapsed": false,
    "executionInfo": {
     "elapsed": 5993,
     "status": "ok",
     "timestamp": 1445965582896,
     "user": {
      "color": "#1FA15D",
      "displayName": "Vincent Vanhoucke",
      "isAnonymous": false,
      "isMe": true,
      "permissionId": "05076109866853157986",
      "photoUrl": "//lh6.googleusercontent.com/-cCJa7dTDcgQ/AAAAAAAAAAI/AAAAAAAACgw/r2EZ_8oYer4/s50-c-k-no/photo.jpg",
      "sessionId": "6f6f07b359200c46",
      "userId": "102167687554210253930"
     },
     "user_tz": 420
    },
    "id": "RJ-o3UBUFtCw",
    "outputId": "d530534e-0791-4a94-ca6d-1c8f1b908a9e"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found and verified text8.zip\n"
     ]
    }
   ],
   "source": [
    "url = 'http://mattmahoney.net/dc/'\n",
    "\n",
    "def maybe_download(filename, expected_bytes):\n",
    "  \"\"\"Download a file if not present, and make sure it's the right size.\"\"\"\n",
    "  if not os.path.exists(filename):\n",
    "    filename, _ = urlretrieve(url + filename, filename)\n",
    "  statinfo = os.stat(filename)\n",
    "  if statinfo.st_size == expected_bytes:\n",
    "    print('Found and verified %s' % filename)\n",
    "  else:\n",
    "    print(statinfo.st_size)\n",
    "    raise Exception(\n",
    "      'Failed to verify ' + filename + '. Can you get to it with a browser?')\n",
    "  return filename\n",
    "\n",
    "filename = maybe_download('text8.zip', 31344016)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "cellView": "both",
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "output_extras": [
      {
       "item_id": 1
      }
     ]
    },
    "colab_type": "code",
    "collapsed": false,
    "executionInfo": {
     "elapsed": 5982,
     "status": "ok",
     "timestamp": 1445965582916,
     "user": {
      "color": "#1FA15D",
      "displayName": "Vincent Vanhoucke",
      "isAnonymous": false,
      "isMe": true,
      "permissionId": "05076109866853157986",
      "photoUrl": "//lh6.googleusercontent.com/-cCJa7dTDcgQ/AAAAAAAAAAI/AAAAAAAACgw/r2EZ_8oYer4/s50-c-k-no/photo.jpg",
      "sessionId": "6f6f07b359200c46",
      "userId": "102167687554210253930"
     },
     "user_tz": 420
    },
    "id": "Mvf09fjugFU_",
    "outputId": "8f75db58-3862-404b-a0c3-799380597390"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Data size 100000000\n"
     ]
    }
   ],
   "source": [
    "def read_data(filename):\n",
    "  f = zipfile.ZipFile(filename)\n",
    "  for name in f.namelist():\n",
    "    return tf.compat.as_str(f.read(name))\n",
    "  f.close()\n",
    "  \n",
    "text = read_data(filename)\n",
    "print('Data size %d' % len(text))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "ga2CYACE-ghb"
   },
   "source": [
    "Create a small validation set."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "cellView": "both",
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "output_extras": [
      {
       "item_id": 1
      }
     ]
    },
    "colab_type": "code",
    "collapsed": false,
    "executionInfo": {
     "elapsed": 6184,
     "status": "ok",
     "timestamp": 1445965583138,
     "user": {
      "color": "#1FA15D",
      "displayName": "Vincent Vanhoucke",
      "isAnonymous": false,
      "isMe": true,
      "permissionId": "05076109866853157986",
      "photoUrl": "//lh6.googleusercontent.com/-cCJa7dTDcgQ/AAAAAAAAAAI/AAAAAAAACgw/r2EZ_8oYer4/s50-c-k-no/photo.jpg",
      "sessionId": "6f6f07b359200c46",
      "userId": "102167687554210253930"
     },
     "user_tz": 420
    },
    "id": "w-oBpfFG-j43",
    "outputId": "bdb96002-d021-4379-f6de-a977924f0d02"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(99999000, 'ons anarchists advocate social relations based upon voluntary association of autonomous individuals mutual aid and self governance while anarchism is most easily defined by what it is against anarchists also offer positive visions of what they believe to be a truly free society however ideas about how an anarchist society might work vary considerably especially with respect to economics there is also disagreement about how a free society might be brought about origins and predecessors kropotkin and others argue that before recorded history human society was organized on anarchist principles most anthropologists follow kropotkin and engels in believing that hunter gatherer bands were egalitarian and lacked division of labour accumulated wealth or decreed law and had equal access to resources william godwin anarchists including the the anarchy organisation and rothbard find anarchist attitudes in taoism from ancient china kropotkin found similar ideas in stoic zeno of citium according to kropotkin zeno repudiat')\n",
      "(1000, ' anarchism originated as a term of abuse first used against early working class radicals including t')\n"
     ]
    }
   ],
   "source": [
    "valid_size = 1000\n",
    "valid_text = text[:valid_size]\n",
    "train_text = text[valid_size:]\n",
    "train_size = len(train_text)\n",
    "print(train_size, train_text[:1024])\n",
    "print(valid_size, valid_text[:100])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "Zdw6i4F8glpp"
   },
   "source": [
    "Utility functions to map characters to vocabulary IDs and back."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "cellView": "both",
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "output_extras": [
      {
       "item_id": 1
      }
     ]
    },
    "colab_type": "code",
    "collapsed": false,
    "executionInfo": {
     "elapsed": 6276,
     "status": "ok",
     "timestamp": 1445965583249,
     "user": {
      "color": "#1FA15D",
      "displayName": "Vincent Vanhoucke",
      "isAnonymous": false,
      "isMe": true,
      "permissionId": "05076109866853157986",
      "photoUrl": "//lh6.googleusercontent.com/-cCJa7dTDcgQ/AAAAAAAAAAI/AAAAAAAACgw/r2EZ_8oYer4/s50-c-k-no/photo.jpg",
      "sessionId": "6f6f07b359200c46",
      "userId": "102167687554210253930"
     },
     "user_tz": 420
    },
    "id": "gAL1EECXeZsD",
    "outputId": "88fc9032-feb9-45ff-a9a0-a26759cc1f2e"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Unexpected character: ï\n",
      "(1, 26, 0, 0)\n",
      "('a', 'z', ' ')\n"
     ]
    }
   ],
   "source": [
    "vocabulary_size = len(string.ascii_lowercase) + 1 # [a-z] + ' '\n",
    "first_letter = ord(string.ascii_lowercase[0])\n",
    "\n",
    "def char2id(char):\n",
    "  if char in string.ascii_lowercase:\n",
    "    return ord(char) - first_letter + 1\n",
    "  elif char == ' ':\n",
    "    return 0\n",
    "  else:\n",
    "    print('Unexpected character: %s' % char)\n",
    "    return 0\n",
    "  \n",
    "def id2char(dictid):\n",
    "  if dictid > 0:\n",
    "    return chr(dictid + first_letter - 1)\n",
    "  else:\n",
    "    return ' '\n",
    "\n",
    "print(char2id('a'), char2id('z'), char2id(' '), char2id('ï'))\n",
    "print(id2char(1), id2char(26), id2char(0))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "lFwoyygOmWsL"
   },
   "source": [
    "Function to generate a training batch for the LSTM model."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "cellView": "both",
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "output_extras": [
      {
       "item_id": 1
      }
     ]
    },
    "colab_type": "code",
    "collapsed": false,
    "executionInfo": {
     "elapsed": 6473,
     "status": "ok",
     "timestamp": 1445965583467,
     "user": {
      "color": "#1FA15D",
      "displayName": "Vincent Vanhoucke",
      "isAnonymous": false,
      "isMe": true,
      "permissionId": "05076109866853157986",
      "photoUrl": "//lh6.googleusercontent.com/-cCJa7dTDcgQ/AAAAAAAAAAI/AAAAAAAACgw/r2EZ_8oYer4/s50-c-k-no/photo.jpg",
      "sessionId": "6f6f07b359200c46",
      "userId": "102167687554210253930"
     },
     "user_tz": 420
    },
    "id": "d9wMtjy5hCj9",
    "outputId": "3dd79c80-454a-4be0-8b71-4a4a357b3367"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1, 781243, 1562485, 2343727, 3124969, 3906211, 4687453, 5468695, 6249937, 7031179, 7812421, 8593663, 9374905, 10156147, 10937389, 11718631, 12499873, 13281115, 14062357, 14843599, 15624841, 16406083, 17187325, 17968567, 18749809, 19531051, 20312293, 21093535, 21874777, 22656019, 23437261, 24218503, 24999745, 25780987, 26562229, 27343471, 28124713, 28905955, 29687197, 30468439, 31249681, 32030923, 32812165, 33593407, 34374649, 35155891, 35937133, 36718375, 37499617, 38280859, 39062101, 39843343, 40624585, 41405827, 42187069, 42968311, 43749553, 44530795, 45312037, 46093279, 46874521, 47655763, 48437005, 49218247, 49999489, 50780731, 51561973, 52343215, 53124457, 53905699, 54686941, 55468183, 56249425, 57030667, 57811909, 58593151, 59374393, 60155635, 60936877, 61718119, 62499361, 63280603, 64061845, 64843087, 65624329, 66405571, 67186813, 67968055, 68749297, 69530539, 70311781, 71093023, 71874265, 72655507, 73436749, 74217991, 74999233, 75780475, 76561717, 77342959, 78124201, 78905443, 79686685, 80467927, 81249169, 82030411, 82811653, 83592895, 84374137, 85155379, 85936621, 86717863, 87499105, 88280347, 89061589, 89842831, 90624073, 91405315, 92186557, 92967799, 93749041, 94530283, 95311525, 96092767, 96874009, 97655251, 98436493, 99217735]\n",
      "['o', 'n', 'w', ' ', 'l', 'r', ' ', 's', 'm', 's', 'h', 'a', 'y', ' ', 'a', 's', 't', 'a', 'm', ' ', 'n', 's', 'h', 's', 'e', 'l', 'e', ' ', 'o', 'l', 'y', 'o', 'o', 'r', 'a', ' ', ' ', 'a', 'a', 'e', 'i', 'e', ' ', 't', 't', 'h', 'd', 'e', 'f', 'f', 'a', 't', 'e', 'i', 'e', 'a', 'a', 'c', 'r', 'g', 'i', ' ', 'o', 'r', 'a', 'f', 'g', 'g', 'i', 'a', 'r', 's', 'c', 'f', 'a', 'e', ' ', 'a', 'm', 'f', 't', 'o', 'u', 'e', 'e', 't', 'o', 'n', 'o', 'e', 's', 'e', 'k', ' ', 'e', ' ', 'w', 'l', 'e', 'e', 't', ' ', 'e', 'n', ' ', 'l', 'i', 'e', 't', 'n', 'd', 's', 't', 'e', 'e', 'r', 'f', 'e', 'd', 'i', 't', 'n', 'a', 'e', 'a', 'r', 's', 'n']\n",
      "['ons anarchists advocate social relations based upon vol', 'nomination gore s endorsement of dean was helpful to th', 'when military governments failed to revive the economy ', ' three nine one six zero two zero zero one census peter', 'lleria arches national park photographic virtual tour o', 'reviated as dr mr and mrs respectively they are also fr', ' abbeys and monasteries index sacred destinations abbey', 'shing the right of appeal to the judicial committee of ', 'married urraca princess of castile daughter of alfonso ', 'sity upset the devils which cost the school its nationa', 'hel and richard baer h provided a detailed description ', 'ased in the st family here they are in rough chronologi', 'y and liturgical language among jews mandaeans and some', ' disgust because of the relationship between the anus a', 'ay opened for passengers in december one nine zero two ', 'society and that this neglect is the true cause of the ', 'tion from the national media and from presidential cand', 'ago based chess records label the influence of blues on', 'migration took place during the one nine eight zero s w', ' zero zero five yaniv shaked and avishai wool published', 'new york other well known manufacturers of bass amplifi', 'short subject college humor one nine three three too mu', 'he boeing seven six seven a widebody jet was introduced', 'sgow two young white men whose murderers were asian and', 'e listed with a gloss covering some of their deeds a si', 'lt during this period however the iran iraq war of the ', 'eber has probably been one of the most influential user', ' not dead naturally and hangs herself upon hearing the ', 'o be made to recognize single acts of merit or meritori', 'll s enthusiastic backing darwin read his first paper t', 'yer who received the first card from the deal may be kn', 'operates three submarines based in talcahuano air force', 'ore significant than in jersey and guernsey has maintai', 'rmines security of the system provided that there is no', 'a fierce critic of the poverty and social stratificatio', ' fuel extracted from the ground by underground mining o', ' two six eight in signs of humanity vol three michel ba', 'ature that was attacking his livestock it was later det', 'aristotle s uncaused cause so aquinas comes to the same', 'e dragas constantine i of imereti constantine iii of ro', 'ity can be lost as in denaturalization and gained as in', 'ecombinant region and the diode becomes conductive whic', ' and intracellular ice formation solution effects are c', 'tensive manufacturing sectors the question of why the m', 'tion of the size of the input usually measured in bits ', 'he attack from hyrsyl northwards and reached petrozavod', 'dy to pass him a stick to pull him out but she refuses ', 'ed to bring good fortune to those who carried them owne', 'f certain drugs confusion inability to orient oneself l', 'french jansenist theologian b one six three four one se', 'at it will take to complete an operation cannot be boun', 'tion from euclidean geometry and analysis such as gradi', 'e convince the priest of the mistakes of a pious life t', 'ither spontaneously or have employed in their daily wri', 'ent told him to name it fort des moines the original or', 'argest partner of the uk has also made it a destination', 'ampaign and barred attempts by his opponents to run cam', 'ce in a special cell named down s cell the cell is conn', 'rver side standard formats for mailboxes include maildi', 'gain the amplified signal from q one is directly fed to', 'ious texts such as esoteric christianity and the work o', ' assignment of numbers to positions a player who is not', 'o capitalize on the growing popularity of disco with th', 'rettas francis poulenc jean philippe rameau maurice rav', 'a duplicate of the original document fax machines with ', 'former is widely used in the internal combustion engine', 'gh ann es d hiver one nine eight zero one nine eight si', 'g the series to a seventh and deciding game the next ni', 'ine january eight march eight listing of all days days ', 'ar it is possible the tradition was carried on by the l', 'ross zero the lead character lieutenant shin kudo playe', 's worldwide after english german is the second largest ', 'cal theories classical mechanics and special relativity', 'f an inch they were fit together so perfectly that the ', 'ast instance the non gm comparison maize crop had also ', 'e phantom appear on stage is pepper s ghost technique i', ' dimensional analysis fundamental applications of proba', 'as pi approx frac the algorithm has second order conver', 'most holy mormons believe the configuration of the cont', 'fice did not become formalized for some decades charlem', 't s support or at least not parliament s opposition a s', 'olunteers originally hand picked who used to do the job', 'u is still disagreed upon by historians and linguists i', 'e end of world war ii and the near extermination of eur', 'e oscillating system example rlc circuit full mathemati', 'tain its surface area over time and potassium which inc', 'o eight subtypes based on the whole genome that are eac', 'normal course of study was abolished the academic cours', 'of italy languages the official language of italy is st', 'e of the leadership according to the law all presidenti', 's the tower commission at this point president reagan s', 'e icj reports seven see charney j compromissory clauses', 'klahoma press one nine three two one one th printing on', ' most of the indictees are serbs the tribunal exclusive', 'erprise linux suse linux enterprise server debian and t', ' running of the household until her death in one seven ', 'ws becomes the first daily college newspaper in the uni', 'lization about the performance of java programs in gene', 'et in a nazi concentration camp lewis has explained why', 'ey accepted a proposal from their relative johann matth', 'the fabian society nehru wished the economy of india to', ' help when the queen left he supposedly noticed her foo', 'etchy to relatively stiff from flat to tightly curled a', 'ng bombs with global positioning system satellite guida', ' sharman networks sharman s sydney based boss nikki hem', 'laces for karaoke and terms of karaoke for a descriptio', 'ised emperor hirohito to begin negotiations to end worl', 'erican football and also called an onside pass in canad', 'ting in political initiatives the lesotho congress for ', 'nd has been awarded the two zero zero six polar music p', 'd neo latin most of these authors wrote in their variou', 'seco the a s took an early lead in game one on a grand ', 'th risky riskerdoo ricky ricardo this classic includes ', 'er donna troy who is destined to play a critical role i', 'encyclopedic overview of mathematics presented in clear', 'rall definition of spheres of influence in the area abs', 'fense the air component of arm is represented by the co', 'ecause of communist regime languages khalkha mongol nin', 'duating from acnm accredited programs must pass the sam', 'ifferent focal planes called a z stack plus the knowled', 'treet grid centerline external links bbc on this day ma', 'n militant union leaderships additionally thatcher s ri', 'ations more than any other state modern day montana bec', 'etal compounds such as cp two ba or structures with ben', 'appeal of devotional buddhism especially represented by', 'rs larry and james one older sister delores and a young', 'si have made such devices possible the systemic advanta', 'ncluding employees of many companies who had replaced t']\n",
      "['luntary association of autonomous individuals mutual ai', 'he latter in legitimizing him in the eyes of the establ', ' and suppress escalating terrorism in the late one nine', 'rhead is the largest town in aberdeenshire the principa', 'of arches national park archaeological sites in the uni', 'requently written as in canada and the u s as dr mr and', 'ys of france sacred destinations abbeys art history the', ' the privy council an evolving independence thus the in', ' viii king of castile and leonora of aquitaine in one t', 'al ranking the wins over washington state and washingto', ' of the camp s workings during his interrogations after', 'ical order after the original five two zero st five two', 'e christians and is still spoken by small isolated comm', 'and feces however it is not uncommon for the rectum to ', ' on the night of friday one four th january one nine th', ' poverty and misery experienced by the residents of tho', 'didate john f kennedy despite this incident atlanta s p', 'n mainstream american popular music was huge in the fif', 'with the arrival of thousands of refugees from guatemal', 'd the paper cracking the bluetooth pin one which shows ', 'iers or loudspeakers include accugroove loudpeakers acm', 'uch harmony one nine three three please one nine three ', 'd at around the same time as the seven five seven its n', 'd whose murders the bnp maintains were hate crimes the ', 'ignificance is attached to the thirty and the three all', ' one nine eight zero s was a difficult time for the cit', 'rs of the word in its social science sense he is well k', ' news discovery and translation modern day picture of t', 'ious service the required achievement or service while ', 'to the geological society of london on four january one', 'nown as eldest hand or as forehand the set of cards dea', 'e fach gen osvaldo sarabia heads a force of one two fiv', 'ined light industry as a higher proportion of its econo', 'o analytic attack i e a structural weakness in the algo', 'on of victorian society throughout his works dickens re', 'or open pit mining strip mining it is a readily combust', 'alat and janice deledalle rhodes eds g rard deledalle g', 'termined to be a canine most likely a coyote of some so', 'e conclusion that god exists whether there was a first ', 'ome constantine mavrocordato constantine nicolaievich c', 'n naturalization supranational citizenship in recent ye', 'ch allows electrons to flow though the diode from the c', 'caused by concentration of solutes in non frozen soluti', 'maritimes fell from being a centre of canadian manufact', ' using the most efficient algorithm to understand this ', 'dsk railroad and the main road the next day from there ', ' unless he declare his devotion to god almighty hume ac', 'ership was restricted among various castes by color wit', 'later signs lethargy decreased ability to perform simpl', 'even two three philip ii duke of orl ans regent of fran', 'nded in advance see unbounded nondeterminism scalabilit', 'ient of a function divergence length of curves and so o', 'the novel the one two zero days of sodom written in one', 'iting to explore themselves and their experience of the', 'rigin of the name des moines is uncertain it could have', 'n for economic migrants from scotland wales and norther', 'mpaign advertisements for this reason many countries en', 'nected to a battery allowing electrons migration from t', 'ir and mbox several prominent e mail clients use their ', 'o the second stage q three which provides further ampli', 'of g i gurdjieff a variety of past traditions could be ', 't wearing a number that corresponds to an eligible rece', 'he album discovery or disco very as he has been quoted ', 'vel claude joseph rouget de lisle composer of la marsei', ' additional electronic features can connect to computer', 'e while both are used in power generation nuclear fuels', 'ix and cartographies schizoanalytique one nine eight ni', 'ight which was won by cincinnati duffy s cliff from one', ' february nine is the four zero th day of the year in t', 'local community after the travellers had left or that l', 'ed by kenichi suzumura is a qualified f one four pilot ', ' of the germanic languages german is the language with ', 'y classical mechanics and special relativity are lumped', ' tip of a knife cannot be inserted between the joints a', ' been treated with environmentally damaging pesticides ', 'in asian horror cinema the ghost stories often include ', 'ability and statistics nine specialized topics nuclear ', 'rgent nature which essentially means that the number of', 'tinents was different before the great flood and that t', 'magne went on to adopt the title augustus from earlier ', 'subtle but important difference it also gives parliamen', 'bs of scouts as well prior to the creation of peer revi', 'it is generally accepted as having originally been a pe', 'ropean jewry by the nazis international support for jew', 'ical definition most harmonic oscillators at least appr', 'creases the electron density of the catalyst and so imp', 'ch geographically distinct the most prevalent are subty', 'se became standard across the student body in one nine ', 'tandard italian a direct descendant of latin some seven', 'ial candidates must be approved by the council of guard', 'said he had not been informed of the operation the towe', 's and the jurisdiction of the international court of ju', 'ne nine eight nine isbn zero eight zero six one one one', 'ely uses translators who speak bosnian and croatian lan', 'the version sgi offers on their altix machines in jan t', ' two nine it was in weimar that two musically significa', 'ited states one eight eight seven in a snowstorm at for', 'eral because run time performance is affected much more', 'y the film hasn t been released by suggesting litigatio', 'hias franck the schoolmaster and choirmaster in hainbur', 'o be partially capitalist but with the state occupying ', 'otprint in the floor plaster of his workplace even thou', 'and so on process a modern knitting machine in action a', 'ance devices that are immune to bad weather also althou', 'mming and associate kevin bermeister had knowingly allo', 'on of karaoke boxes in two zero zero four daisuke inoue', 'ld war ii after the beginning of the american occupatio', 'dian football is a sideways or rearward throwing of the', ' democracy lcd won the majority in parliament in the tw', 'prize in rolling stone magazine s tabulation of the one', 'us vernaculars as well as in latin but each produced a ', ' slam by canseco and led four three in the bottom of th', ' lucy winding the cello s tuning peg as if it were a wa', 'in infinite crisis he s also been carefully surveilling', 'r simple language hazewinkel michiel ed encyclopaedia o', 'solute french control over madagascar was established b', 'ommand of military aviation and air defense of the repu', 'ne zero turkic qazaq tuvin russian literacy definition ', 'me certifying exam administered by the american midwife', 'dge of the psf which can be either derived experimental', 'ay two seven may two nine april two eight june two eigh', 'ight to buy scheme to buy policy whereby council housin', 'came montana territory in one eight six four by appoint', 'nt aromatic rings such as found in manganocene or titan', 'y the pure land this rich cosmography also allowed maha', 'ger sister roslyn he married juanita jordan in septembe', 'ages of mpls such as the ability to support multiple se', 'their pension systems with annuities purchased from the']\n",
      "[' a']\n",
      "['an']\n"
     ]
    }
   ],
   "source": [
    "class BatchGenerator(object):\n",
    "  def __init__(self, text, batch_size, num_unrollings):\n",
    "    self._text = text\n",
    "    self._text_size = len(text)\n",
    "    self._batch_size = batch_size\n",
    "    self._num_unrollings = num_unrollings\n",
    "    segment = self._text_size // batch_size\n",
    "    self._cursor = [ offset * segment for offset in range(batch_size)]\n",
    "    self._last_batch = self._next_batch()\n",
    "  \n",
    "  def _next_batch(self):\n",
    "    \"\"\"Generate a single batch from the current cursor position in the data.\"\"\"\n",
    "    batch = np.zeros(shape=(self._batch_size, vocabulary_size), dtype=np.float)\n",
    "    #print 'batch idx %i' % \n",
    "    for b in range(self._batch_size):\n",
    "      batch[b, char2id(self._text[self._cursor[b]])] = 1.0\n",
    "      self._cursor[b] = (self._cursor[b] + 1) % self._text_size\n",
    "    return batch\n",
    "  \n",
    "  def next(self):\n",
    "    \"\"\"Generate the next array of batches from the data. The array consists of\n",
    "    the last batch of the previous array, followed by num_unrollings new ones.\n",
    "    \"\"\"\n",
    "    batches = [self._last_batch]\n",
    "    for step in range(self._num_unrollings):\n",
    "      batches.append(self._next_batch())\n",
    "    self._last_batch = batches[-1]\n",
    "    return batches\n",
    "\n",
    "def characters(probabilities):\n",
    "  \"\"\"Turn a 1-hot encoding or a probability distribution over the possible\n",
    "  characters back into its (mostl likely) character representation.\"\"\"\n",
    "  return [id2char(c) for c in np.argmax(probabilities, 1)]\n",
    "\n",
    "def batches2string(batches):\n",
    "  \"\"\"Convert a sequence of batches back into their (most likely) string\n",
    "  representation.\"\"\"\n",
    "  s = [''] * batches[0].shape[0]\n",
    "  for b in batches:\n",
    "    s = [''.join(x) for x in zip(s, characters(b))]\n",
    "  return s\n",
    "\n",
    "train_batches = BatchGenerator(train_text, 128, 54)\n",
    "valid_batches = BatchGenerator(valid_text, 1, 1)\n",
    "print train_batches._cursor\n",
    "\n",
    "batch = train_batches.next()\n",
    "print batches2string([batch[0]])\n",
    "print(batches2string(batch))\n",
    "print(batches2string(train_batches.next()))\n",
    "print(batches2string(valid_batches.next()))\n",
    "print(batches2string(valid_batches.next()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[array([419, 419, 522,  36,  47, 540, 221, 490]), array([513,   6, 203, 378, 135, 423,   6, 420]), array([ 41, 501, 249, 126, 411,   1, 261,  18]), array([ 45, 351, 246, 574,  20, 540, 533, 246]), array([ 89, 548,  41, 513, 221, 329,   4, 322]), array([262, 135, 540,  96,  15,  46,  36,  18]), array([559, 379, 548, 384, 495, 540, 349, 246]), array([  1, 549,  41, 586, 198, 393,   3,  45]), array([130, 419,   9, 258, 379, 540, 417, 123])]\n",
      "['ons anarchists adv', 'on from the nation', 'significant than i', 'ain drugs confusio', 'ate of the origina', 't or at least not ', 'he first daily col', 'rdoo ricky ricardo']\n",
      "[array([[ 0.,  0.,  0., ...,  0.,  0.,  0.],\n",
      "       [ 0.,  0.,  0., ...,  0.,  0.,  0.],\n",
      "       [ 0.,  0.,  0., ...,  0.,  0.,  0.],\n",
      "       ..., \n",
      "       [ 0.,  0.,  0., ...,  0.,  0.,  0.],\n",
      "       [ 0.,  0.,  0., ...,  0.,  0.,  0.],\n",
      "       [ 0.,  0.,  0., ...,  0.,  0.,  0.]]), array([[ 0.,  0.,  0., ...,  0.,  0.,  0.],\n",
      "       [ 0.,  0.,  0., ...,  0.,  0.,  0.],\n",
      "       [ 0.,  0.,  0., ...,  0.,  0.,  0.],\n",
      "       ..., \n",
      "       [ 0.,  0.,  0., ...,  0.,  0.,  0.],\n",
      "       [ 0.,  0.,  0., ...,  0.,  0.,  0.],\n",
      "       [ 0.,  0.,  0., ...,  0.,  0.,  0.]]), array([[ 0.,  0.,  0., ...,  0.,  0.,  0.],\n",
      "       [ 0.,  0.,  0., ...,  0.,  0.,  0.],\n",
      "       [ 0.,  0.,  0., ...,  0.,  0.,  0.],\n",
      "       ..., \n",
      "       [ 0.,  1.,  0., ...,  0.,  0.,  0.],\n",
      "       [ 0.,  0.,  0., ...,  0.,  0.,  0.],\n",
      "       [ 0.,  0.,  0., ...,  0.,  0.,  0.]]), array([[ 0.,  0.,  0., ...,  0.,  0.,  0.],\n",
      "       [ 0.,  0.,  0., ...,  0.,  0.,  0.],\n",
      "       [ 0.,  0.,  0., ...,  0.,  0.,  0.],\n",
      "       ..., \n",
      "       [ 0.,  0.,  0., ...,  0.,  0.,  0.],\n",
      "       [ 0.,  0.,  0., ...,  0.,  0.,  0.],\n",
      "       [ 0.,  0.,  0., ...,  0.,  0.,  0.]]), array([[ 0.,  0.,  0., ...,  0.,  0.,  0.],\n",
      "       [ 0.,  0.,  0., ...,  0.,  0.,  0.],\n",
      "       [ 0.,  0.,  0., ...,  0.,  0.,  0.],\n",
      "       ..., \n",
      "       [ 0.,  0.,  0., ...,  0.,  0.,  0.],\n",
      "       [ 0.,  0.,  0., ...,  0.,  0.,  0.],\n",
      "       [ 0.,  0.,  0., ...,  0.,  0.,  0.]]), array([[ 0.,  0.,  0., ...,  0.,  0.,  0.],\n",
      "       [ 0.,  0.,  0., ...,  0.,  0.,  0.],\n",
      "       [ 0.,  0.,  0., ...,  0.,  0.,  0.],\n",
      "       ..., \n",
      "       [ 0.,  0.,  0., ...,  0.,  0.,  0.],\n",
      "       [ 0.,  0.,  0., ...,  0.,  0.,  0.],\n",
      "       [ 0.,  0.,  0., ...,  0.,  0.,  0.]]), array([[ 0.,  0.,  0., ...,  0.,  0.,  0.],\n",
      "       [ 0.,  0.,  0., ...,  0.,  0.,  0.],\n",
      "       [ 0.,  0.,  0., ...,  0.,  0.,  0.],\n",
      "       ..., \n",
      "       [ 0.,  0.,  0., ...,  0.,  0.,  0.],\n",
      "       [ 0.,  0.,  0., ...,  0.,  0.,  0.],\n",
      "       [ 0.,  0.,  0., ...,  0.,  0.,  0.]]), array([[ 0.,  1.,  0., ...,  0.,  0.,  0.],\n",
      "       [ 0.,  0.,  0., ...,  0.,  0.,  0.],\n",
      "       [ 0.,  0.,  0., ...,  0.,  0.,  0.],\n",
      "       ..., \n",
      "       [ 0.,  0.,  0., ...,  0.,  0.,  0.],\n",
      "       [ 0.,  0.,  0., ...,  0.,  0.,  0.],\n",
      "       [ 0.,  0.,  0., ...,  0.,  0.,  0.]]), array([[ 0.,  0.,  0., ...,  0.,  0.,  0.],\n",
      "       [ 0.,  0.,  0., ...,  0.,  0.,  0.],\n",
      "       [ 0.,  0.,  0., ...,  0.,  0.,  0.],\n",
      "       ..., \n",
      "       [ 0.,  0.,  0., ...,  0.,  0.,  0.],\n",
      "       [ 0.,  0.,  0., ...,  0.,  0.,  0.],\n",
      "       [ 0.,  0.,  0., ...,  0.,  0.,  0.]])]\n",
      "['dvocate social rel', 'onal media and fro', ' in jersey and gue', 'ion inability to o', 'nal document fax m', 't parliament s opp', 'ollege newspaper i', 'do this classic in']\n",
      "[' ana']\n",
      "['narc']\n"
     ]
    }
   ],
   "source": [
    "bi_voc_size = vocabulary_size * vocabulary_size\n",
    "\n",
    "class BiBatchGenerator(object):\n",
    "  def __init__(self, text, batch_size, num_unrollings):\n",
    "    self._text = text\n",
    "    self._text_size_in_chars = len(text)\n",
    "    self._text_size = self._text_size_in_chars // 2 # in bigrams\n",
    "    self._batch_size = batch_size\n",
    "    self._num_unrollings = num_unrollings\n",
    "    segment = self._text_size // batch_size\n",
    "    self._cursor = [ offset * segment for offset in range(batch_size)]\n",
    "    self._last_batch = self._next_batch()\n",
    "  \n",
    "  def _next_batch(self):    \n",
    "    batch = np.zeros(shape=(self._batch_size), dtype=np.int)\n",
    "    #print 'batch idx %i' % \n",
    "    for b in range(self._batch_size):\n",
    "      char_idx = self._cursor[b]*2\n",
    "      ch1 = char2id(self._text[char_idx])\n",
    "      if(self._text_size_in_chars - 1 == char_idx):\n",
    "            ch2 = 0\n",
    "      else:\n",
    "            ch2 = char2id(self._text[char_idx + 1])\n",
    "      batch[b] = ch1*vocabulary_size + ch2\n",
    "      self._cursor[b] = (self._cursor[b] + 1) % self._text_size\n",
    "    return batch\n",
    "  \n",
    "  def next(self):    \n",
    "    batches = [self._last_batch]\n",
    "    for step in range(self._num_unrollings):\n",
    "      batches.append(self._next_batch())\n",
    "    self._last_batch = batches[-1]\n",
    "    return batches\n",
    "\n",
    "def bi2str(encoding):\n",
    "    return id2char(encoding // vocabulary_size ) + id2char(encoding % vocabulary_size)\n",
    "\n",
    "def bigrams(encodings):\n",
    "  return [bi2str(e) for e in encodings]\n",
    "\n",
    "def bibatches2string(batches):\n",
    "  s = [''] * batches[0].shape[0]\n",
    "  for b in batches:\n",
    "    s = [''.join(x) for x in zip(s, bigrams(b))]\n",
    "  return s\n",
    "\n",
    "bi_onehot = np.zeros((bi_voc_size, bi_voc_size))\n",
    "np.fill_diagonal(bi_onehot, 1)\n",
    "def bigramonehot(encodings):\n",
    "    return [bi_onehot[e] for e in encodings]\n",
    "\n",
    "train_batches = BiBatchGenerator(train_text, 8, 8)\n",
    "valid_batches = BiBatchGenerator(valid_text, 1, 1)\n",
    "\n",
    "batch = train_batches.next()\n",
    "print batch\n",
    "print bibatches2string(batch)\n",
    "#print bigramonehot(batch)\n",
    "print bibatches2string(train_batches.next())\n",
    "print bibatches2string(valid_batches.next())\n",
    "print bibatches2string(valid_batches.next())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "cellView": "both",
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": true,
    "id": "KyVd8FxT5QBc"
   },
   "outputs": [],
   "source": [
    "def logprob(predictions, labels):\n",
    "  \"\"\"Log-probability of the true labels in a predicted batch.\"\"\"\n",
    "  predictions[predictions < 1e-10] = 1e-10\n",
    "  return np.sum(np.multiply(labels, -np.log(predictions))) / labels.shape[0]\n",
    "\n",
    "def sample_distribution(distribution):\n",
    "  \"\"\"Sample one element from a distribution assumed to be an array of normalized\n",
    "  probabilities.\n",
    "  \"\"\"\n",
    "  r = random.uniform(0, 1)\n",
    "  s = 0\n",
    "  for i in range(len(distribution)):\n",
    "    s += distribution[i]\n",
    "    if s >= r:\n",
    "      return i\n",
    "  return len(distribution) - 1\n",
    "\n",
    "def sample(prediction, size=vocabulary_size):\n",
    "  \"\"\"Turn a (column) prediction into 1-hot encoded samples.\"\"\"\n",
    "  p = np.zeros(shape=[1, size], dtype=np.float)\n",
    "  p[0, sample_distribution(prediction[0])] = 1.0\n",
    "  return p\n",
    "\n",
    "def random_distribution(size=vocabulary_size):\n",
    "  \"\"\"Generate a random column of probabilities.\"\"\"\n",
    "  b = np.random.uniform(0.0, 1.0, size=[1, size])\n",
    "  return b/np.sum(b, 1)[:,None]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "K8f67YXaDr4C"
   },
   "source": [
    "Simple LSTM Model."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "cellView": "both",
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "collapsed": false,
    "id": "Q5rxZK6RDuGe"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "logits:\n",
      "Tensor(\"xw_plus_b:0\", shape=TensorShape([Dimension(640), Dimension(27)]), dtype=float32)\n",
      "loss:\n",
      "Tensor(\"loss:0\", shape=TensorShape([]), dtype=float32)\n",
      "<tensorflow.python.training.gradient_descent.GradientDescentOptimizer object at 0x7fd20e8eced0>\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.framework.ops.Graph at 0x7fd20eb71910>"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def create_lstm_graph(num_nodes, num_unrollings, batch_size):\n",
    "    with tf.Graph().as_default() as g:\n",
    "      # Parameters:\n",
    "      # Input gate: input, previous output, and bias.\n",
    "      ix = tf.Variable(tf.truncated_normal([vocabulary_size, num_nodes], -0.1, 0.1), name='ix')\n",
    "      im = tf.Variable(tf.truncated_normal([num_nodes, num_nodes], -0.1, 0.1))\n",
    "      ib = tf.Variable(tf.zeros([1, num_nodes]))\n",
    "      # Forget gate: input, previous output, and bias.\n",
    "      fx = tf.Variable(tf.truncated_normal([vocabulary_size, num_nodes], -0.1, 0.1))\n",
    "      fm = tf.Variable(tf.truncated_normal([num_nodes, num_nodes], -0.1, 0.1))\n",
    "      fb = tf.Variable(tf.zeros([1, num_nodes]))\n",
    "      # Memory cell: input, state and bias.                             \n",
    "      cx = tf.Variable(tf.truncated_normal([vocabulary_size, num_nodes], -0.1, 0.1))\n",
    "      cm = tf.Variable(tf.truncated_normal([num_nodes, num_nodes], -0.1, 0.1))\n",
    "      cb = tf.Variable(tf.zeros([1, num_nodes]))\n",
    "      # Output gate: input, previous output, and bias.\n",
    "      ox = tf.Variable(tf.truncated_normal([vocabulary_size, num_nodes], -0.1, 0.1))\n",
    "      om = tf.Variable(tf.truncated_normal([num_nodes, num_nodes], -0.1, 0.1))\n",
    "      ob = tf.Variable(tf.zeros([1, num_nodes]))\n",
    "      # Variables saving state across unrollings.\n",
    "      saved_output = tf.Variable(tf.zeros([batch_size, num_nodes]), trainable=False)\n",
    "      saved_state = tf.Variable(tf.zeros([batch_size, num_nodes]), trainable=False)\n",
    "      # Classifier weights and biases.\n",
    "      w = tf.Variable(tf.truncated_normal([num_nodes, vocabulary_size], -0.1, 0.1))\n",
    "      b = tf.Variable(tf.zeros([vocabulary_size]))\n",
    "\n",
    "      # Definition of the cell computation.\n",
    "      def lstm_cell(i, o, state):\n",
    "        \"\"\"Create a LSTM cell. See e.g.: http://arxiv.org/pdf/1402.1128v1.pdf\n",
    "        Note that in this formulation, we omit the various connections between the\n",
    "        previous state and the gates.\"\"\"\n",
    "        input_gate = tf.sigmoid(tf.matmul(i, ix) + tf.matmul(o, im) + ib)\n",
    "        forget_gate = tf.sigmoid(tf.matmul(i, fx) + tf.matmul(o, fm) + fb)\n",
    "        update = tf.matmul(i, cx) + tf.matmul(o, cm) + cb\n",
    "        state = forget_gate * state + input_gate * tf.tanh(update)\n",
    "        output_gate = tf.sigmoid(tf.matmul(i, ox) + tf.matmul(o, om) + ob)\n",
    "        return output_gate * tf.tanh(state), state\n",
    "\n",
    "      # Input data. [num_unrollings, batch_size, vocabulary_size]\n",
    "      tf_train_data = tf.placeholder(tf.float32, shape=[None, None, vocabulary_size], name='tf_train_data')\n",
    "      train_data = list()\n",
    "      for i in tf.split(0, num_unrollings + 1, tf_train_data):\n",
    "        train_data.append(tf.squeeze(i))\n",
    "      train_inputs = train_data[:num_unrollings]\n",
    "      train_labels = train_data[1:]  # labels are inputs shifted by one time step.\n",
    "\n",
    "      # Unrolled LSTM loop.\n",
    "      outputs = list()\n",
    "      output = saved_output\n",
    "      state = saved_state\n",
    "      #python loop used: tensorflow does not support sequential operations yet\n",
    "      for i in train_inputs: # having a loop simulates having time\n",
    "        output, state = lstm_cell(i, output, state)\n",
    "        outputs.append(output)\n",
    "\n",
    "      # State saving across unrollings, control_dependencies makes sure that output and state are computed\n",
    "      with tf.control_dependencies([saved_output.assign(output), saved_state.assign(state)]):\n",
    "        # Classifier.\n",
    "        logits = tf.nn.xw_plus_b(tf.concat(0, outputs), w, b)\n",
    "        print 'logits:'\n",
    "        print logits\n",
    "        loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits, tf.concat(0, train_labels)),\n",
    "                             name='loss')\n",
    "        print 'loss:'\n",
    "        print loss\n",
    "\n",
    "      # Optimizer.\n",
    "      global_step = tf.Variable(0, name='global_step')\n",
    "      learning_rate = tf.train.exponential_decay(10.0, global_step, 5000, 0.1, staircase=True, name='learning_rate')\n",
    "      optimizer = tf.train.GradientDescentOptimizer(learning_rate, name='optimizer')\n",
    "      print optimizer\n",
    "      gradients, v = zip(*optimizer.compute_gradients(loss))\n",
    "      gradients, _ = tf.clip_by_global_norm(gradients, 1.25)\n",
    "      optimizer = optimizer.apply_gradients(zip(gradients, v), global_step=global_step)\n",
    "\n",
    "      # Predictions.\n",
    "      train_prediction = tf.nn.softmax(logits, name='train_prediction')\n",
    "\n",
    "      # Sampling and validation eval: batch 1, no unrolling.\n",
    "      sample_input = tf.placeholder(tf.float32, shape=[1, vocabulary_size], name='sample_input')\n",
    "      saved_sample_output = tf.Variable(tf.zeros([1, num_nodes]), name='saved_sample_output')\n",
    "      saved_sample_state = tf.Variable(tf.zeros([1, num_nodes]), name='saved_sample_state')\n",
    "      reset_sample_state = tf.group( saved_sample_output.assign(tf.zeros([1, num_nodes])),\n",
    "                            saved_sample_state.assign(tf.zeros([1, num_nodes])), name='reset_sample_state')\n",
    "      sample_output, sample_state = lstm_cell(\n",
    "        sample_input, saved_sample_output, saved_sample_state)\n",
    "      with tf.control_dependencies([saved_sample_output.assign(sample_output),\n",
    "                                    saved_sample_state.assign(sample_state)]):\n",
    "        sample_prediction = tf.nn.softmax(tf.nn.xw_plus_b(sample_output, w, b), name='sample_prediction')\n",
    "      \n",
    "      return g\n",
    "\n",
    "#test graph\n",
    "create_lstm_graph(64, 10, 64)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "cellView": "both",
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "output_extras": [
      {
       "item_id": 41
      },
      {
       "item_id": 80
      },
      {
       "item_id": 126
      },
      {
       "item_id": 144
      }
     ]
    },
    "colab_type": "code",
    "collapsed": false,
    "executionInfo": {
     "elapsed": 199909,
     "status": "ok",
     "timestamp": 1445965877333,
     "user": {
      "color": "#1FA15D",
      "displayName": "Vincent Vanhoucke",
      "isAnonymous": false,
      "isMe": true,
      "permissionId": "05076109866853157986",
      "photoUrl": "//lh6.googleusercontent.com/-cCJa7dTDcgQ/AAAAAAAAAAI/AAAAAAAACgw/r2EZ_8oYer4/s50-c-k-no/photo.jpg",
      "sessionId": "6f6f07b359200c46",
      "userId": "102167687554210253930"
     },
     "user_tz": 420
    },
    "id": "RD9zQCZTEaEm",
    "outputId": "5e868466-2532-4545-ce35-b403cf5d9de6",
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "def train(g, num_steps, summary_frequency, num_unrollings, batch_size):\n",
    "    #initalize batch generators\n",
    "    train_batches = BatchGenerator(train_text, batch_size, num_unrollings)\n",
    "    valid_batches = BatchGenerator(valid_text, 1, 1)\n",
    "    optimizer = g.get_tensor_by_name('optimizer:0')\n",
    "    #print optimizer\n",
    "    loss = g.get_tensor_by_name('loss:0')\n",
    "    train_prediction = g.get_tensor_by_name('train_prediction:0')\n",
    "    learning_rate = g.get_tensor_by_name('learning_rate:0')\n",
    "    tf_train_data = g.get_tensor_by_name('tf_train_data:0')\n",
    "    sample_prediction = g.get_tensor_by_name('sample_prediction:0')\n",
    "    reset_sample_state = g.get_operation_by_name('reset_sample_state')\n",
    "    sample_input = g.get_tensor_by_name('sample_input:0')\n",
    "    with tf.Session(graph=g) as session:\n",
    "      tf.initialize_all_variables().run()\n",
    "      print('Initialized')    \n",
    "      mean_loss = 0\n",
    "      for step in range(num_steps):\n",
    "        batches = train_batches.next()\n",
    "        #print np.array(batches)\n",
    "        #feed_dict = dict()\n",
    "        #for i in range(num_unrollings + 1):\n",
    "        #  feed_dict[train_data[i]] = batches[i]            \n",
    "        #tf_train_data = \n",
    "        _, l, predictions, lr = session.run([optimizer, loss, train_prediction, learning_rate], \n",
    "                                            feed_dict={ tf_train_data: batches})\n",
    "        mean_loss += 1\n",
    "        if step % summary_frequency == 0:\n",
    "          if step > 0:\n",
    "            mean_loss = mean_loss / summary_frequency\n",
    "          # The mean loss is an estimate of the loss over the last few batches.\n",
    "          #print mean_loss\n",
    "          #print type(mean_loss)\n",
    "          print 'Average loss at step %d: %f learning rate: %f' % (step, mean_loss, lr)\n",
    "          mean_loss = 0\n",
    "          labels = np.concatenate(list(batches)[1:])\n",
    "          print('Minibatch perplexity: %.2f' % float(np.exp(logprob(predictions, labels))))\n",
    "          if step % (summary_frequency * 10) == 0:\n",
    "            # Generate some samples.\n",
    "            print('=' * 80)\n",
    "            for _ in range(5):\n",
    "              feed = sample(random_distribution())\n",
    "              sentence = characters(feed)[0]\n",
    "              reset_sample_state.run()\n",
    "              for _ in range(79):\n",
    "                prediction = sample_prediction.eval({sample_input: feed})\n",
    "                feed = sample(prediction)\n",
    "                sentence += characters(feed)[0]\n",
    "              print(sentence)\n",
    "            print('=' * 80)\n",
    "          # Measure validation set perplexity.\n",
    "          reset_sample_state.run()\n",
    "          valid_logprob = 0\n",
    "          for _ in range(valid_size):\n",
    "            b = valid_batches.next()\n",
    "            predictions = sample_prediction.eval({sample_input: b[0]})\n",
    "            valid_logprob = valid_logprob + logprob(predictions, b[1])\n",
    "          print('Validation set perplexity: %.2f' % float(np.exp(\n",
    "            valid_logprob / valid_size)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<tensorflow.python.training.gradient_descent.GradientDescentOptimizer object at 0x7f5144e04710>\n",
      "Initialized\n",
      "Average loss at step 0: 1.000000 learning rate: 10.000000\n",
      "Minibatch perplexity: 27.16\n",
      "================================================================================\n",
      "yhdo t o  araxh kegsnymotkrren xvw foinor ankequlhb vnvbgae pllwf uvctr yztr fci\n",
      "bmvp u mgdshltdhaxdplnz eh itcrvstualluzu  feqyesra eoiesabjypii ooa tkptcyslgoe\n",
      "s wnvsybxuw hwai ipiezabz hx lio  f t  rlancamdp cpsnk  ga zphppfsjeewwhsqzrrlr \n",
      "l avkxnkfwnasktiw yty dddqbqsyvhmeffaatprla proh tm sdpvatitcar bs  ziigvlqqxies\n",
      "s lmwprjje lf  khtizuamlflo nuguvuer  zfxgr  aej  flrjxfadrasyqzhkecxe  mw etki \n",
      "================================================================================\n",
      "Validation set perplexity: 20.10\n",
      "Average loss at step 100: 1.000000 learning rate: 10.000000\n",
      "Minibatch perplexity: 10.27\n",
      "Validation set perplexity: 10.26\n",
      "Average loss at step 200: 1.000000 learning rate: 10.000000\n",
      "Minibatch perplexity: 9.03\n",
      "Validation set perplexity: 8.75\n",
      "Average loss at step 300: 1.000000 learning rate: 10.000000\n",
      "Minibatch perplexity: 7.15\n",
      "Validation set perplexity: 8.05\n",
      "Average loss at step 400: 1.000000 learning rate: 10.000000\n",
      "Minibatch perplexity: 7.34\n",
      "Validation set perplexity: 7.32\n",
      "Average loss at step 500: 1.000000 learning rate: 10.000000\n",
      "Minibatch perplexity: 6.37\n",
      "Validation set perplexity: 6.85\n",
      "Average loss at step 600: 1.000000 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.44\n",
      "Validation set perplexity: 6.46\n",
      "Average loss at step 700: 1.000000 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.87\n",
      "Validation set perplexity: 6.08\n",
      "Average loss at step 800: 1.000000 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.41\n",
      "Validation set perplexity: 5.73\n",
      "Average loss at step 900: 1.000000 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.85\n",
      "Validation set perplexity: 5.42\n",
      "Average loss at step 1000: 1.000000 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.66\n",
      "================================================================================\n",
      "qurphistion maunogla acholy part tennoled iblo lood who thon wate moshanda it is\n",
      "jan disbeppand lume it carced in arlivem to distans theiustally moks crastly of \n",
      "bul how impuenly as that the reforpiyolachent lankes an orquds that the explood \n",
      "x proyssers or acters fas smstems coathos roquadb grompical systems mecomenty co\n",
      "get or hold viuds han folmoth censa gion the farls arrica in kusone called from \n",
      "================================================================================\n",
      "Validation set perplexity: 5.11\n",
      "Average loss at step 1100: 1.000000 learning rate: 10.000000\n",
      "Minibatch perplexity: 4.90\n",
      "Validation set perplexity: 4.95\n",
      "Average loss at step 1200: 1.000000 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.62\n",
      "Validation set perplexity: 4.80\n",
      "Average loss at step 1300: 1.000000 learning rate: 10.000000\n",
      "Minibatch perplexity: 4.83\n",
      "Validation set perplexity: 4.63\n",
      "Average loss at step 1400: 1.000000 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.24\n",
      "Validation set perplexity: 4.64\n",
      "Average loss at step 1500: 1.000000 learning rate: 10.000000\n",
      "Minibatch perplexity: 4.97\n",
      "Validation set perplexity: 4.49\n",
      "Average loss at step 1600: 1.000000 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.21\n",
      "Validation set perplexity: 4.24\n",
      "Average loss at step 1700: 1.000000 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.06\n",
      "Validation set perplexity: 4.31\n",
      "Average loss at step 1800: 1.000000 learning rate: 10.000000\n",
      "Minibatch perplexity: 4.97\n",
      "Validation set perplexity: 4.33\n",
      "Average loss at step 1900: 1.000000 learning rate: 10.000000\n",
      "Minibatch perplexity: 4.74\n",
      "Validation set perplexity: 4.23\n",
      "Average loss at step 2000: 1.000000 learning rate: 10.000000\n",
      "Minibatch perplexity: 4.59\n",
      "================================================================================\n",
      "x as white history the consplemations and aware exter beckepball sound the ad su\n",
      "mentor to eowancs and airmage leading in refied autles this niging hendern dame \n",
      "t three eight plinspersisuds suds links a drevetor shistian c indialing instrunc\n",
      "ch webe presirns of counter maluer polin recians to larian the year been the fiv\n",
      "f for johnies the of sho leves pained as have other one nine frue ows develope a\n",
      "================================================================================\n",
      "Validation set perplexity: 4.28\n",
      "Average loss at step 2100: 1.000000 learning rate: 10.000000\n",
      "Minibatch perplexity: 4.66\n",
      "Validation set perplexity: 4.21\n",
      "Average loss at step 2200: 1.000000 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.32\n",
      "Validation set perplexity: 4.17\n",
      "Average loss at step 2300: 1.000000 learning rate: 10.000000\n",
      "Minibatch perplexity: 4.66\n",
      "Validation set perplexity: 4.13\n",
      "Average loss at step 2400: 1.000000 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.17\n",
      "Validation set perplexity: 4.13\n",
      "Average loss at step 2500: 1.000000 learning rate: 10.000000\n",
      "Minibatch perplexity: 4.54\n",
      "Validation set perplexity: 4.08\n",
      "Average loss at step 2600: 1.000000 learning rate: 10.000000\n",
      "Minibatch perplexity: 4.59\n",
      "Validation set perplexity: 4.07\n",
      "Average loss at step 2700: 1.000000 learning rate: 10.000000\n",
      "Minibatch perplexity: 4.57\n",
      "Validation set perplexity: 4.01\n",
      "Average loss at step 2800: 1.000000 learning rate: 10.000000\n",
      "Minibatch perplexity: 4.81\n",
      "Validation set perplexity: 4.06\n",
      "Average loss at step 2900: 1.000000 learning rate: 10.000000\n",
      "Minibatch perplexity: 4.27\n",
      "Validation set perplexity: 3.98\n",
      "Average loss at step 3000: 1.000000 learning rate: 10.000000\n",
      "Minibatch perplexity: 4.71\n",
      "================================================================================\n",
      "king of view as positively of forces althoughere row exolosing delicity con jaqu\n",
      "che one sinneding an implems in deteles avossessence and specials was to fureent\n",
      "on his forcs of evill d clas bundh were syrno telk money bacly oavismorie only g\n",
      "alls it diely team it ell groupher of genations that to b one seven eight intern\n",
      "quaters government in muile settled and s tradiog one eight six four moyern kigh\n",
      "================================================================================\n",
      "Validation set perplexity: 4.11\n",
      "Average loss at step 3100: 1.000000 learning rate: 10.000000\n",
      "Minibatch perplexity: 4.79\n",
      "Validation set perplexity: 4.00\n",
      "Average loss at step 3200: 1.000000 learning rate: 10.000000\n",
      "Minibatch perplexity: 4.16\n",
      "Validation set perplexity: 3.90\n",
      "Average loss at step 3300: 1.000000 learning rate: 10.000000\n",
      "Minibatch perplexity: 4.86\n",
      "Validation set perplexity: 4.01\n",
      "Average loss at step 3400: 1.000000 learning rate: 10.000000\n",
      "Minibatch perplexity: 4.13\n",
      "Validation set perplexity: 3.89\n",
      "Average loss at step 3500: 1.000000 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.23\n",
      "Validation set perplexity: 4.04\n",
      "Average loss at step 3600: 1.000000 learning rate: 10.000000\n",
      "Minibatch perplexity: 4.32\n",
      "Validation set perplexity: 3.97\n",
      "Average loss at step 3700: 1.000000 learning rate: 10.000000\n",
      "Minibatch perplexity: 3.97\n",
      "Validation set perplexity: 4.00\n",
      "Average loss at step 3800: 1.000000 learning rate: 10.000000\n",
      "Minibatch perplexity: 4.56\n",
      "Validation set perplexity: 4.06\n",
      "Average loss at step 3900: 1.000000 learning rate: 10.000000\n",
      "Minibatch perplexity: 4.05\n",
      "Validation set perplexity: 3.97\n",
      "Average loss at step 4000: 1.000000 learning rate: 10.000000\n",
      "Minibatch perplexity: 4.15\n",
      "================================================================================\n",
      "m one nine eight three one eight subdars that to coarpha shi stranges day diving\n",
      "ison d council absosted cered contropo was palone is technation again zero of an\n",
      "maty which companient wowles under arbamble for infentions st at also belier or \n",
      "p unite one zero comotite pover one newsuarized country risting oencally the daw\n",
      "berning thus kround string over when kaong ad the one nine three one five eight \n",
      "================================================================================\n",
      "Validation set perplexity: 3.99\n",
      "Average loss at step 4100: 1.000000 learning rate: 10.000000\n",
      "Minibatch perplexity: 4.43\n",
      "Validation set perplexity: 4.05\n",
      "Average loss at step 4200: 1.000000 learning rate: 10.000000\n",
      "Minibatch perplexity: 4.51\n",
      "Validation set perplexity: 3.98\n",
      "Average loss at step 4300: 1.000000 learning rate: 10.000000\n",
      "Minibatch perplexity: 3.98\n",
      "Validation set perplexity: 4.01\n",
      "Average loss at step 4400: 1.000000 learning rate: 10.000000\n",
      "Minibatch perplexity: 4.54\n",
      "Validation set perplexity: 4.03\n",
      "Average loss at step 4500: 1.000000 learning rate: 10.000000\n",
      "Minibatch perplexity: 3.83\n",
      "Validation set perplexity: 4.01\n",
      "Average loss at step 4600: 1.000000 learning rate: 10.000000\n",
      "Minibatch perplexity: 4.00\n",
      "Validation set perplexity: 3.97\n",
      "Average loss at step 4700: 1.000000 learning rate: 10.000000\n",
      "Minibatch perplexity: 4.10\n",
      "Validation set perplexity: 3.95\n",
      "Average loss at step 4800: 1.000000 learning rate: 10.000000\n",
      "Minibatch perplexity: 3.99\n",
      "Validation set perplexity: 3.87\n",
      "Average loss at step 4900: 1.000000 learning rate: 10.000000\n",
      "Minibatch perplexity: 4.26\n",
      "Validation set perplexity: 4.06\n",
      "Average loss at step 5000: 1.000000 learning rate: 1.000000\n",
      "Minibatch perplexity: 4.47\n",
      "================================================================================\n",
      "zquion in thefere in storistal give and aghert upnett sone in novelover years of\n",
      "zers in europen os cammon new jarioaks agreer porters with it entrapority of fut\n",
      "is see and loss evidence lobetism real bytraval later file by all painting it ra\n",
      "dically byoaking cames d one seven reternalistic governher in polyttratives d on\n",
      "zer aclaed furcue and orience to the sh bba to bune american china two and tibik\n",
      "================================================================================\n",
      "Validation set perplexity: 3.87\n",
      "Average loss at step 5100: 1.000000 learning rate: 1.000000\n",
      "Minibatch perplexity: 4.85\n",
      "Validation set perplexity: 3.84\n",
      "Average loss at step 5200: 1.000000 learning rate: 1.000000\n",
      "Minibatch perplexity: 4.58\n",
      "Validation set perplexity: 3.81\n",
      "Average loss at step 5300: 1.000000 learning rate: 1.000000\n",
      "Minibatch perplexity: 4.01\n",
      "Validation set perplexity: 3.79\n",
      "Average loss at step 5400: 1.000000 learning rate: 1.000000\n",
      "Minibatch perplexity: 4.65\n",
      "Validation set perplexity: 3.79\n",
      "Average loss at step 5500: 1.000000 learning rate: 1.000000\n",
      "Minibatch perplexity: 4.17\n",
      "Validation set perplexity: 3.74\n",
      "Average loss at step 5600: 1.000000 learning rate: 1.000000\n",
      "Minibatch perplexity: 3.75\n",
      "Validation set perplexity: 3.74\n",
      "Average loss at step 5700: 1.000000 learning rate: 1.000000\n",
      "Minibatch perplexity: 3.93\n",
      "Validation set perplexity: 3.75\n",
      "Average loss at step 5800: 1.000000 learning rate: 1.000000\n",
      "Minibatch perplexity: 4.92\n",
      "Validation set perplexity: 3.73\n",
      "Average loss at step 5900: 1.000000 learning rate: 1.000000\n",
      "Minibatch perplexity: 4.24\n",
      "Validation set perplexity: 3.74\n",
      "Average loss at step 6000: 1.000000 learning rate: 1.000000\n",
      "Minibatch perplexity: 4.02\n",
      "================================================================================\n",
      "s leory having is not very below mainfordey tepenary respectively four balffay a\n",
      "urge affacer the jchoses and the roman who army a lessolukaid one nine two five \n",
      "uguest process a marta drized order king protacburion that possible one nine eig\n",
      "verly nowel gaired structures for polent allease was that incorporative line of \n",
      "ents of rage four apmanypooms who centranopern ridy in orla g l one eight four e\n",
      "================================================================================\n",
      "Validation set perplexity: 3.73\n",
      "Average loss at step 6100: 1.000000 learning rate: 1.000000\n",
      "Minibatch perplexity: 4.66\n",
      "Validation set perplexity: 3.74\n",
      "Average loss at step 6200: 1.000000 learning rate: 1.000000\n",
      "Minibatch perplexity: 4.75\n",
      "Validation set perplexity: 3.74\n",
      "Average loss at step 6300: 1.000000 learning rate: 1.000000\n",
      "Minibatch perplexity: 4.25\n",
      "Validation set perplexity: 3.74\n",
      "Average loss at step 6400: 1.000000 learning rate: 1.000000\n",
      "Minibatch perplexity: 4.27\n",
      "Validation set perplexity: 3.73\n",
      "Average loss at step 6500: 1.000000 learning rate: 1.000000\n",
      "Minibatch perplexity: 4.43\n",
      "Validation set perplexity: 3.76\n",
      "Average loss at step 6600: 1.000000 learning rate: 1.000000\n",
      "Minibatch perplexity: 4.19\n",
      "Validation set perplexity: 3.78\n",
      "Average loss at step 6700: 1.000000 learning rate: 1.000000\n",
      "Minibatch perplexity: 4.88\n",
      "Validation set perplexity: 3.75\n",
      "Average loss at step 6800: 1.000000 learning rate: 1.000000\n",
      "Minibatch perplexity: 4.29\n",
      "Validation set perplexity: 3.74\n",
      "Average loss at step 6900: 1.000000 learning rate: 1.000000\n",
      "Minibatch perplexity: 3.89\n",
      "Validation set perplexity: 3.71\n",
      "Average loss at step 7000: 1.000000 learning rate: 1.000000\n",
      "Minibatch perplexity: 4.08\n",
      "================================================================================\n",
      "el term two zero zero four and nevices eight three paralius ablevent he was real\n",
      "inated alfon four faurism most to objecial regreje ending two devolutepria s gin\n",
      "and clouds war bausees in a one nine which in the two zero zero classism from on\n",
      "hel cities of being on americans have commonley workhoid of tanks of commont fou\n",
      "qu one nine three three paintries which indietation of was use featurmyan loze c\n",
      "================================================================================\n",
      "Validation set perplexity: 3.71\n",
      "Average loss at step 7100: 1.000000 learning rate: 1.000000\n",
      "Minibatch perplexity: 3.98\n",
      "Validation set perplexity: 3.73\n",
      "Average loss at step 7200: 1.000000 learning rate: 1.000000\n",
      "Minibatch perplexity: 4.08\n",
      "Validation set perplexity: 3.72\n",
      "Average loss at step 7300: 1.000000 learning rate: 1.000000\n",
      "Minibatch perplexity: 3.96\n",
      "Validation set perplexity: 3.74\n",
      "Average loss at step 7400: 1.000000 learning rate: 1.000000\n",
      "Minibatch perplexity: 4.20\n",
      "Validation set perplexity: 3.73\n",
      "Average loss at step 7500: 1.000000 learning rate: 1.000000\n",
      "Minibatch perplexity: 4.07\n",
      "Validation set perplexity: 3.72\n",
      "Average loss at step 7600: 1.000000 learning rate: 1.000000\n",
      "Minibatch perplexity: 4.12\n",
      "Validation set perplexity: 3.73\n",
      "Average loss at step 7700: 1.000000 learning rate: 1.000000\n",
      "Minibatch perplexity: 4.51\n",
      "Validation set perplexity: 3.77\n",
      "Average loss at step 7800: 1.000000 learning rate: 1.000000\n",
      "Minibatch perplexity: 3.95\n",
      "Validation set perplexity: 3.75\n",
      "Average loss at step 7900: 1.000000 learning rate: 1.000000\n",
      "Minibatch perplexity: 4.57\n",
      "Validation set perplexity: 3.72\n",
      "Average loss at step 8000: 1.000000 learning rate: 1.000000\n",
      "Minibatch perplexity: 4.32\n",
      "================================================================================\n",
      "y of proveds energype called like serbs to release to meetamarchive guitation fo\n",
      "qude it was the could kread the people service there is not location of the mill\n",
      "untimate battex of justive nat three but argue have allow claudions of the took \n",
      "ause of man or one one nine five in one nine nine five etcrofilly the venish fiv\n",
      "kertovist b of functioned in autes their news kerght national end have opposed t\n",
      "================================================================================\n",
      "Validation set perplexity: 3.69\n",
      "Average loss at step 8100: 1.000000 learning rate: 1.000000\n",
      "Minibatch perplexity: 3.63\n",
      "Validation set perplexity: 3.70\n",
      "Average loss at step 8200: 1.000000 learning rate: 1.000000\n",
      "Minibatch perplexity: 4.24\n",
      "Validation set perplexity: 3.73\n",
      "Average loss at step 8300: 1.000000 learning rate: 1.000000\n",
      "Minibatch perplexity: 3.92\n",
      "Validation set perplexity: 3.77\n",
      "Average loss at step 8400: 1.000000 learning rate: 1.000000\n",
      "Minibatch perplexity: 3.64\n",
      "Validation set perplexity: 3.78\n",
      "Average loss at step 8500: 1.000000 learning rate: 1.000000\n",
      "Minibatch perplexity: 3.81\n",
      "Validation set perplexity: 3.77\n",
      "Average loss at step 8600: 1.000000 learning rate: 1.000000\n",
      "Minibatch perplexity: 3.59\n",
      "Validation set perplexity: 3.76\n",
      "Average loss at step 8700: 1.000000 learning rate: 1.000000\n",
      "Minibatch perplexity: 4.62\n",
      "Validation set perplexity: 3.75\n",
      "Average loss at step 8800: 1.000000 learning rate: 1.000000\n",
      "Minibatch perplexity: 4.93\n",
      "Validation set perplexity: 3.76\n",
      "Average loss at step 8900: 1.000000 learning rate: 1.000000\n",
      "Minibatch perplexity: 4.35\n",
      "Validation set perplexity: 3.75\n",
      "Average loss at step 9000: 1.000000 learning rate: 1.000000\n",
      "Minibatch perplexity: 4.13\n",
      "================================================================================\n",
      "urderal what be women than a convention which consmice but texchings and adds to\n",
      "anded tlommini his isste as this growing both of the wordmunn emidpic cave the t\n",
      "charine religions and nlissical reliting dubatted in an one nine nine two sp tco\n",
      "ther projects for artagiation of the christ becaugh it western one nine four fou\n",
      "m didwestips which celts measure by alget and several powers the bay beander tha\n",
      "================================================================================\n",
      "Validation set perplexity: 3.75\n",
      "Average loss at step 9100: 1.000000 learning rate: 1.000000\n",
      "Minibatch perplexity: 4.33\n",
      "Validation set perplexity: 3.76\n",
      "Average loss at step 9200: 1.000000 learning rate: 1.000000\n",
      "Minibatch perplexity: 4.11\n",
      "Validation set perplexity: 3.78\n",
      "Average loss at step 9300: 1.000000 learning rate: 1.000000\n",
      "Minibatch perplexity: 4.29\n",
      "Validation set perplexity: 3.77\n",
      "Average loss at step 9400: 1.000000 learning rate: 1.000000\n",
      "Minibatch perplexity: 3.76\n",
      "Validation set perplexity: 3.78\n",
      "Average loss at step 9500: 1.000000 learning rate: 1.000000\n",
      "Minibatch perplexity: 3.97\n",
      "Validation set perplexity: 3.80\n",
      "Average loss at step 9600: 1.000000 learning rate: 1.000000\n",
      "Minibatch perplexity: 4.32\n",
      "Validation set perplexity: 3.81\n",
      "Average loss at step 9700: 1.000000 learning rate: 1.000000\n",
      "Minibatch perplexity: 3.96\n",
      "Validation set perplexity: 3.77\n",
      "Average loss at step 9800: 1.000000 learning rate: 1.000000\n",
      "Minibatch perplexity: 4.60\n",
      "Validation set perplexity: 3.80\n",
      "Average loss at step 9900: 1.000000 learning rate: 1.000000\n",
      "Minibatch perplexity: 4.14\n",
      "Validation set perplexity: 3.77\n",
      "Average loss at step 10000: 1.000000 learning rate: 0.100000\n",
      "Minibatch perplexity: 4.23\n",
      "================================================================================\n",
      "d four another paturals albanible orway particle in unsadive girrogillay mitter \n",
      "gety one ove while bascrogimal prestorated it israel center competing the tear a\n",
      "kings etraining through he rather for the never like own area have to polias one\n",
      "pearation music plane family a dward with stai signal group caunified the victoo\n",
      "jest and one three zero two sequence in argadd fled cellicary chemicalipality sp\n",
      "================================================================================\n",
      "Validation set perplexity: 3.78\n",
      "Average loss at step 10100: 1.000000 learning rate: 0.100000\n",
      "Minibatch perplexity: 3.93\n",
      "Validation set perplexity: 3.78\n",
      "Average loss at step 10200: 1.000000 learning rate: 0.100000\n",
      "Minibatch perplexity: 3.86\n",
      "Validation set perplexity: 3.78\n",
      "Average loss at step 10300: 1.000000 learning rate: 0.100000\n",
      "Minibatch perplexity: 4.15\n",
      "Validation set perplexity: 3.77\n",
      "Average loss at step 10400: 1.000000 learning rate: 0.100000\n",
      "Minibatch perplexity: 4.56\n",
      "Validation set perplexity: 3.77\n",
      "Average loss at step 10500: 1.000000 learning rate: 0.100000\n",
      "Minibatch perplexity: 4.46\n",
      "Validation set perplexity: 3.77\n",
      "Average loss at step 10600: 1.000000 learning rate: 0.100000\n",
      "Minibatch perplexity: 4.81\n",
      "Validation set perplexity: 3.76\n",
      "Average loss at step 10700: 1.000000 learning rate: 0.100000\n",
      "Minibatch perplexity: 4.11\n",
      "Validation set perplexity: 3.75\n",
      "Average loss at step 10800: 1.000000 learning rate: 0.100000\n",
      "Minibatch perplexity: 4.28\n",
      "Validation set perplexity: 3.75\n",
      "Average loss at step 10900: 1.000000 learning rate: 0.100000\n",
      "Minibatch perplexity: 4.16\n",
      "Validation set perplexity: 3.75\n",
      "Average loss at step 11000: 1.000000 learning rate: 0.100000\n",
      "Minibatch perplexity: 4.20\n",
      "================================================================================\n",
      "zer s europet request in the most national is universe usevections on custon ser\n",
      "cean disk quanbersons had germone exactives of interismer colors hybrows if noha\n",
      "ke trunctus naturalism christ and coalf chain the exmvation and est off pajes of\n",
      "bilim three also elizab us bely court refiued ow edward days to will andellova l\n",
      "eans alish minor pacies of unized by upons of mu colonia chanfall by peninience \n",
      "================================================================================\n",
      "Validation set perplexity: 3.76\n",
      "Average loss at step 11100: 1.000000 learning rate: 0.100000\n",
      "Minibatch perplexity: 4.26\n",
      "Validation set perplexity: 3.75\n",
      "Average loss at step 11200: 1.000000 learning rate: 0.100000\n",
      "Minibatch perplexity: 4.79\n",
      "Validation set perplexity: 3.75\n",
      "Average loss at step 11300: 1.000000 learning rate: 0.100000\n",
      "Minibatch perplexity: 4.52\n",
      "Validation set perplexity: 3.75\n",
      "Average loss at step 11400: 1.000000 learning rate: 0.100000\n",
      "Minibatch perplexity: 4.09\n",
      "Validation set perplexity: 3.75\n",
      "Average loss at step 11500: 1.000000 learning rate: 0.100000\n",
      "Minibatch perplexity: 4.32\n",
      "Validation set perplexity: 3.75\n",
      "Average loss at step 11600: 1.000000 learning rate: 0.100000\n",
      "Minibatch perplexity: 3.83\n",
      "Validation set perplexity: 3.76\n",
      "Average loss at step 11700: 1.000000 learning rate: 0.100000\n",
      "Minibatch perplexity: 4.18\n",
      "Validation set perplexity: 3.75\n",
      "Average loss at step 11800: 1.000000 learning rate: 0.100000\n",
      "Minibatch perplexity: 3.92\n",
      "Validation set perplexity: 3.75\n",
      "Average loss at step 11900: 1.000000 learning rate: 0.100000\n",
      "Minibatch perplexity: 4.84\n",
      "Validation set perplexity: 3.75\n",
      "Average loss at step 12000: 1.000000 learning rate: 0.100000\n",
      "Minibatch perplexity: 3.71\n",
      "================================================================================\n",
      "dionia was iranese maximame likely of beating and level separding sometimes or a\n",
      " monitet this increased in groups from sold infriful that been university that t\n",
      "n up organization myth d one eight six zero six luck when his two zero zero thre\n",
      "thing with sficku deactichly still in sembly towest among the single one nine fi\n",
      "quesine intersunger mission of his over ristrails in a seawarmore they the natio\n",
      "================================================================================\n",
      "Validation set perplexity: 3.75\n",
      "Average loss at step 12100: 1.000000 learning rate: 0.100000\n",
      "Minibatch perplexity: 4.33\n",
      "Validation set perplexity: 3.74\n",
      "Average loss at step 12200: 1.000000 learning rate: 0.100000\n",
      "Minibatch perplexity: 4.31\n",
      "Validation set perplexity: 3.74\n",
      "Average loss at step 12300: 1.000000 learning rate: 0.100000\n",
      "Minibatch perplexity: 3.94\n",
      "Validation set perplexity: 3.74\n",
      "Average loss at step 12400: 1.000000 learning rate: 0.100000\n",
      "Minibatch perplexity: 4.03\n",
      "Validation set perplexity: 3.74\n",
      "Average loss at step 12500: 1.000000 learning rate: 0.100000\n",
      "Minibatch perplexity: 4.24\n",
      "Validation set perplexity: 3.74\n",
      "Average loss at step 12600: 1.000000 learning rate: 0.100000\n",
      "Minibatch perplexity: 4.18\n",
      "Validation set perplexity: 3.74\n",
      "Average loss at step 12700: 1.000000 learning rate: 0.100000\n",
      "Minibatch perplexity: 4.82\n",
      "Validation set perplexity: 3.74\n",
      "Average loss at step 12800: 1.000000 learning rate: 0.100000\n",
      "Minibatch perplexity: 4.55\n",
      "Validation set perplexity: 3.74\n",
      "Average loss at step 12900: 1.000000 learning rate: 0.100000\n",
      "Minibatch perplexity: 3.71\n",
      "Validation set perplexity: 3.73\n",
      "Average loss at step 13000: 1.000000 learning rate: 0.100000\n",
      "Minibatch perplexity: 3.99\n",
      "================================================================================\n",
      "compposition are never torate hisstion conditional actor response in more carlit\n",
      "x training the lock the steeric band had the horesinfanta as a metabioned glava \n",
      "remalipics the nomactician before a manjot sentifications one charbed earlte is \n",
      "d of the war international century kintomies on riftents and classics tmin from \n",
      "kritorsiney g b one rook eight two two three subted ny to one two three zero one\n",
      "================================================================================\n",
      "Validation set perplexity: 3.74\n",
      "Average loss at step 13100: 1.000000 learning rate: 0.100000\n",
      "Minibatch perplexity: 3.76\n",
      "Validation set perplexity: 3.74\n",
      "Average loss at step 13200: 1.000000 learning rate: 0.100000\n",
      "Minibatch perplexity: 4.19\n",
      "Validation set perplexity: 3.73\n",
      "Average loss at step 13300: 1.000000 learning rate: 0.100000\n",
      "Minibatch perplexity: 3.76\n",
      "Validation set perplexity: 3.73\n",
      "Average loss at step 13400: 1.000000 learning rate: 0.100000\n",
      "Minibatch perplexity: 4.18\n",
      "Validation set perplexity: 3.73\n",
      "Average loss at step 13500: 1.000000 learning rate: 0.100000\n",
      "Minibatch perplexity: 3.67\n",
      "Validation set perplexity: 3.72\n",
      "Average loss at step 13600: 1.000000 learning rate: 0.100000\n",
      "Minibatch perplexity: 4.13\n",
      "Validation set perplexity: 3.72\n",
      "Average loss at step 13700: 1.000000 learning rate: 0.100000\n",
      "Minibatch perplexity: 4.13\n",
      "Validation set perplexity: 3.72\n",
      "Average loss at step 13800: 1.000000 learning rate: 0.100000\n",
      "Minibatch perplexity: 3.90\n",
      "Validation set perplexity: 3.71\n",
      "Average loss at step 13900: 1.000000 learning rate: 0.100000\n",
      "Minibatch perplexity: 4.05\n",
      "Validation set perplexity: 3.70\n",
      "Average loss at step 14000: 1.000000 learning rate: 0.100000\n",
      "Minibatch perplexity: 3.86\n",
      "================================================================================\n",
      "mings foolstau is as ecv mears san the hervel some seat this remoin and fire the\n",
      "genetics isbn translation four real marine an intelloganance about three four in\n",
      "harments their right transact towards order and is to operated on innfumed statu\n",
      "on morine c duff out germania mimans goes after it when media english a kecinisp\n",
      "i on that the kings of echipselega died if executing this is western but that th\n",
      "================================================================================\n",
      "Validation set perplexity: 3.71\n",
      "Average loss at step 14100: 1.000000 learning rate: 0.100000\n",
      "Minibatch perplexity: 4.46\n",
      "Validation set perplexity: 3.70\n",
      "Average loss at step 14200: 1.000000 learning rate: 0.100000\n",
      "Minibatch perplexity: 4.08\n",
      "Validation set perplexity: 3.70\n",
      "Average loss at step 14300: 1.000000 learning rate: 0.100000\n",
      "Minibatch perplexity: 3.95\n",
      "Validation set perplexity: 3.69\n",
      "Average loss at step 14400: 1.000000 learning rate: 0.100000\n",
      "Minibatch perplexity: 4.16\n",
      "Validation set perplexity: 3.69\n",
      "Average loss at step 14500: 1.000000 learning rate: 0.100000\n",
      "Minibatch perplexity: 4.40\n",
      "Validation set perplexity: 3.68\n",
      "Average loss at step 14600: 1.000000 learning rate: 0.100000\n",
      "Minibatch perplexity: 4.35\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 14700: 1.000000 learning rate: 0.100000\n",
      "Minibatch perplexity: 3.84\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 14800: 1.000000 learning rate: 0.100000\n",
      "Minibatch perplexity: 4.29\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 14900: 1.000000 learning rate: 0.100000\n",
      "Minibatch perplexity: 3.85\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 15000: 1.000000 learning rate: 0.010000\n",
      "Minibatch perplexity: 4.32\n",
      "================================================================================\n",
      "lime inflation which is had current recording hoogs armiation languosts and ruth\n",
      "nesom are a cecavians metupls for arriving an one nine two zero one five five si\n",
      "d warfice of woom external dic voiveis asay probablic oversizer a howest and org\n",
      "hand than the consisting as have novel autonorwowkes todies of in the zature the\n",
      "m when in the now region one methodwain the seuth then one eight six two n l the\n",
      "================================================================================\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 15100: 1.000000 learning rate: 0.010000\n",
      "Minibatch perplexity: 4.38\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 15200: 1.000000 learning rate: 0.010000\n",
      "Minibatch perplexity: 4.21\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 15300: 1.000000 learning rate: 0.010000\n",
      "Minibatch perplexity: 3.82\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 15400: 1.000000 learning rate: 0.010000\n",
      "Minibatch perplexity: 3.91\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 15500: 1.000000 learning rate: 0.010000\n",
      "Minibatch perplexity: 4.20\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 15600: 1.000000 learning rate: 0.010000\n",
      "Minibatch perplexity: 4.19\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 15700: 1.000000 learning rate: 0.010000\n",
      "Minibatch perplexity: 4.56\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 15800: 1.000000 learning rate: 0.010000\n",
      "Minibatch perplexity: 3.63\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 15900: 1.000000 learning rate: 0.010000\n",
      "Minibatch perplexity: 3.65\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 16000: 1.000000 learning rate: 0.010000\n",
      "Minibatch perplexity: 3.97\n",
      "================================================================================\n",
      "her air plain visit one nine eight nine entire neight and cyorkest made believes\n",
      "jo is arbitrie x court involves during the impty of herboard out the browsond in\n",
      "up mans briti or clause of pater eight not royano see and howoth land as from ge\n",
      "main was society assim onation churcolops and develoment on the commentary in th\n",
      "boa two zero zero three subrelied see clayer to portugue racological pht and pla\n",
      "================================================================================\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 16100: 1.000000 learning rate: 0.010000\n",
      "Minibatch perplexity: 4.10\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 16200: 1.000000 learning rate: 0.010000\n",
      "Minibatch perplexity: 3.97\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 16300: 1.000000 learning rate: 0.010000\n",
      "Minibatch perplexity: 3.99\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 16400: 1.000000 learning rate: 0.010000\n",
      "Minibatch perplexity: 4.33\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 16500: 1.000000 learning rate: 0.010000\n",
      "Minibatch perplexity: 3.84\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 16600: 1.000000 learning rate: 0.010000\n",
      "Minibatch perplexity: 3.89\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 16700: 1.000000 learning rate: 0.010000\n",
      "Minibatch perplexity: 4.27\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 16800: 1.000000 learning rate: 0.010000\n",
      "Minibatch perplexity: 4.61\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 16900: 1.000000 learning rate: 0.010000\n",
      "Minibatch perplexity: 4.17\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 17000: 1.000000 learning rate: 0.010000\n",
      "Minibatch perplexity: 4.26\n",
      "================================================================================\n",
      "x goldbeld appear b f dw feta anard lefe five zero elegied in the make ny king h\n",
      "ed to halle see alschipt cent come on batins fuctured such as a soigh carta in o\n",
      "ficial lists macidrers god by therefromer mach trading reinquince daychets opjos\n",
      "most spote the stores vacimate and like puremance highnam cros divident s wasive\n",
      "m as and british and serious least for example because one created the councils \n",
      "================================================================================\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 17100: 1.000000 learning rate: 0.010000\n",
      "Minibatch perplexity: 4.11\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 17200: 1.000000 learning rate: 0.010000\n",
      "Minibatch perplexity: 4.28\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 17300: 1.000000 learning rate: 0.010000\n",
      "Minibatch perplexity: 4.63\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 17400: 1.000000 learning rate: 0.010000\n",
      "Minibatch perplexity: 4.70\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 17500: 1.000000 learning rate: 0.010000\n",
      "Minibatch perplexity: 4.82\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 17600: 1.000000 learning rate: 0.010000\n",
      "Minibatch perplexity: 4.20\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 17700: 1.000000 learning rate: 0.010000\n",
      "Minibatch perplexity: 4.10\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 17800: 1.000000 learning rate: 0.010000\n",
      "Minibatch perplexity: 4.25\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 17900: 1.000000 learning rate: 0.010000\n",
      "Minibatch perplexity: 4.57\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 18000: 1.000000 learning rate: 0.010000\n",
      "Minibatch perplexity: 4.20\n",
      "================================================================================\n",
      "gen industrion gotlosing authon connection reproductions political internal peak\n",
      "ally nyed by a salix kika compozer he leader d one nine seven four eight spinnin\n",
      "red secinariat long changre ousibilibed carricadolo remussine concern more film \n",
      "hobs righers or malays and each making the most convert of collable casts of far\n",
      "unarcologicalyt one three one seven public politicips it entirely of bather the \n",
      "================================================================================\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 18100: 1.000000 learning rate: 0.010000\n",
      "Minibatch perplexity: 4.13\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 18200: 1.000000 learning rate: 0.010000\n",
      "Minibatch perplexity: 4.12\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 18300: 1.000000 learning rate: 0.010000\n",
      "Minibatch perplexity: 4.26\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 18400: 1.000000 learning rate: 0.010000\n",
      "Minibatch perplexity: 4.12\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 18500: 1.000000 learning rate: 0.010000\n",
      "Minibatch perplexity: 4.20\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 18600: 1.000000 learning rate: 0.010000\n",
      "Minibatch perplexity: 4.07\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 18700: 1.000000 learning rate: 0.010000\n",
      "Minibatch perplexity: 4.06\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 18800: 1.000000 learning rate: 0.010000\n",
      "Minibatch perplexity: 4.05\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 18900: 1.000000 learning rate: 0.010000\n",
      "Minibatch perplexity: 4.07\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 19000: 1.000000 learning rate: 0.010000\n",
      "Minibatch perplexity: 4.19\n",
      "================================================================================\n",
      "mer jave appears it userating amoum ah and parx inchinist demand the team set ra\n",
      "ber second the e visubers victique is notworaty on accusingly one nine to the re\n",
      "berly africa abdiginatine croduces arishmin employing although tidy which carbop\n",
      "hamas by acquber nine three five york windock speen evil r elinated and place us\n",
      " linscalane man busine mauszord theories of the modurar and microquikend detroit\n",
      "================================================================================\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 19100: 1.000000 learning rate: 0.010000\n",
      "Minibatch perplexity: 4.86\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 19200: 1.000000 learning rate: 0.010000\n",
      "Minibatch perplexity: 4.09\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 19300: 1.000000 learning rate: 0.010000\n",
      "Minibatch perplexity: 3.65\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 19400: 1.000000 learning rate: 0.010000\n",
      "Minibatch perplexity: 3.81\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 19500: 1.000000 learning rate: 0.010000\n",
      "Minibatch perplexity: 3.88\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 19600: 1.000000 learning rate: 0.010000\n",
      "Minibatch perplexity: 4.78\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 19700: 1.000000 learning rate: 0.010000\n",
      "Minibatch perplexity: 3.73\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 19800: 1.000000 learning rate: 0.010000\n",
      "Minibatch perplexity: 3.93\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 19900: 1.000000 learning rate: 0.010000\n",
      "Minibatch perplexity: 4.12\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 20000: 1.000000 learning rate: 0.001000\n",
      "Minibatch perplexity: 3.96\n",
      "================================================================================\n",
      "laboland nomenzaing laval yueniraphia air fullers raining alogiret the pornived \n",
      "zers programmer net leagt india could offensible bedo genbe spadin of where natu\n",
      "xs and world island a some prodected songs if the guide thiets five lorke contro\n",
      "quious maintainsts are lei part of general british developed etc concentation de\n",
      "ing technicanan asslowor work as was the life legard drowed for the the house hi\n",
      "================================================================================\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 20100: 1.000000 learning rate: 0.001000\n",
      "Minibatch perplexity: 4.14\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 20200: 1.000000 learning rate: 0.001000\n",
      "Minibatch perplexity: 3.86\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 20300: 1.000000 learning rate: 0.001000\n",
      "Minibatch perplexity: 4.43\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 20400: 1.000000 learning rate: 0.001000\n",
      "Minibatch perplexity: 4.39\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 20500: 1.000000 learning rate: 0.001000\n",
      "Minibatch perplexity: 3.84\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 20600: 1.000000 learning rate: 0.001000\n",
      "Minibatch perplexity: 3.99\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 20700: 1.000000 learning rate: 0.001000\n",
      "Minibatch perplexity: 4.21\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 20800: 1.000000 learning rate: 0.001000\n",
      "Minibatch perplexity: 3.77\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 20900: 1.000000 learning rate: 0.001000\n",
      "Minibatch perplexity: 4.13\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 21000: 1.000000 learning rate: 0.001000\n",
      "Minibatch perplexity: 3.88\n",
      "================================================================================\n",
      "gam were they is the change by the remicties detror of good dolver s rook serati\n",
      "le jbhans convention white since time thinds continuental just and meantrrator v\n",
      "anta hell at the government changes by appears can on copure under broad fach at\n",
      "wand on the original linkened not the phyplous of the playes norma who dispuncin\n",
      " and were norther newly ostoca namizations of the term for fook by the yeur onic\n",
      "================================================================================\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 21100: 1.000000 learning rate: 0.001000\n",
      "Minibatch perplexity: 4.06\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 21200: 1.000000 learning rate: 0.001000\n",
      "Minibatch perplexity: 4.30\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 21300: 1.000000 learning rate: 0.001000\n",
      "Minibatch perplexity: 3.72\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 21400: 1.000000 learning rate: 0.001000\n",
      "Minibatch perplexity: 4.20\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 21500: 1.000000 learning rate: 0.001000\n",
      "Minibatch perplexity: 4.39\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 21600: 1.000000 learning rate: 0.001000\n",
      "Minibatch perplexity: 4.53\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 21700: 1.000000 learning rate: 0.001000\n",
      "Minibatch perplexity: 4.10\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 21800: 1.000000 learning rate: 0.001000\n",
      "Minibatch perplexity: 4.02\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 21900: 1.000000 learning rate: 0.001000\n",
      "Minibatch perplexity: 3.99\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 22000: 1.000000 learning rate: 0.001000\n",
      "Minibatch perplexity: 4.11\n",
      "================================================================================\n",
      "quing the region a stand of walken two zero zero three to nine gomber of natives\n",
      "bove womkn wemsed he canada pirpo absucts getry was langioust and piladly obyerc\n",
      "manonicate all the novel changes by corts of because of the cutienism themsels w\n",
      "bury philipher prime populary namr sugficit prisons the scould convince be favor\n",
      "warliet the contemporary atmonden aireai moquation les mass would revorce ojeral\n",
      "================================================================================\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 22100: 1.000000 learning rate: 0.001000\n",
      "Minibatch perplexity: 4.15\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 22200: 1.000000 learning rate: 0.001000\n",
      "Minibatch perplexity: 4.17\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 22300: 1.000000 learning rate: 0.001000\n",
      "Minibatch perplexity: 4.41\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 22400: 1.000000 learning rate: 0.001000\n",
      "Minibatch perplexity: 4.40\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 22500: 1.000000 learning rate: 0.001000\n",
      "Minibatch perplexity: 4.28\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 22600: 1.000000 learning rate: 0.001000\n",
      "Minibatch perplexity: 3.87\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 22700: 1.000000 learning rate: 0.001000\n",
      "Minibatch perplexity: 3.99\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 22800: 1.000000 learning rate: 0.001000\n",
      "Minibatch perplexity: 4.49\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 22900: 1.000000 learning rate: 0.001000\n",
      "Minibatch perplexity: 4.17\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 23000: 1.000000 learning rate: 0.001000\n",
      "Minibatch perplexity: 4.40\n",
      "================================================================================\n",
      "podic upris p two as is the field soundoicha has played president de was nothed \n",
      "mamesho lenetimed in the caverlable sandu of the color calendals has not cannas \n",
      "ving black barking the poodal chapail other of charaction of the three ruman the\n",
      "ject one nine six zero top one eight three six one nine seven four and parts alv\n",
      " site curreny force a gatflegms it was involving presenated to par remain mid th\n",
      "================================================================================\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 23100: 1.000000 learning rate: 0.001000\n",
      "Minibatch perplexity: 4.56\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 23200: 1.000000 learning rate: 0.001000\n",
      "Minibatch perplexity: 4.54\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 23300: 1.000000 learning rate: 0.001000\n",
      "Minibatch perplexity: 4.21\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 23400: 1.000000 learning rate: 0.001000\n",
      "Minibatch perplexity: 4.44\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 23500: 1.000000 learning rate: 0.001000\n",
      "Minibatch perplexity: 4.48\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 23600: 1.000000 learning rate: 0.001000\n",
      "Minibatch perplexity: 4.75\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 23700: 1.000000 learning rate: 0.001000\n",
      "Minibatch perplexity: 4.15\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 23800: 1.000000 learning rate: 0.001000\n",
      "Minibatch perplexity: 4.80\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 23900: 1.000000 learning rate: 0.001000\n",
      "Minibatch perplexity: 3.97\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 24000: 1.000000 learning rate: 0.001000\n",
      "Minibatch perplexity: 4.44\n",
      "================================================================================\n",
      "s corage taxani ballion it is the lyan lavea however mijou line of the simbling \n",
      "y runs dictionare a mnd be lennos will out the body challengic edvatorstic field\n",
      "wels the anamop to be by strictip rath paral mediafal and later in confirs at se\n",
      "ch for offering cach populated his most belown it metper cype romancent slavent \n",
      "fer dispited that imill ciomers russian august thatupous is that between three t\n",
      "================================================================================\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 24100: 1.000000 learning rate: 0.001000\n",
      "Minibatch perplexity: 4.02\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 24200: 1.000000 learning rate: 0.001000\n",
      "Minibatch perplexity: 4.34\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 24300: 1.000000 learning rate: 0.001000\n",
      "Minibatch perplexity: 3.81\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 24400: 1.000000 learning rate: 0.001000\n",
      "Minibatch perplexity: 4.33\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 24500: 1.000000 learning rate: 0.001000\n",
      "Minibatch perplexity: 4.28\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 24600: 1.000000 learning rate: 0.001000\n",
      "Minibatch perplexity: 4.35\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 24700: 1.000000 learning rate: 0.001000\n",
      "Minibatch perplexity: 4.68\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 24800: 1.000000 learning rate: 0.001000\n",
      "Minibatch perplexity: 4.32\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 24900: 1.000000 learning rate: 0.001000\n",
      "Minibatch perplexity: 4.39\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 25000: 1.000000 learning rate: 0.000100\n",
      "Minibatch perplexity: 3.96\n",
      "================================================================================\n",
      "dividian sister to drove was divocine brens and because was iii is disint many l\n",
      "y of after the first six two five system photo birthsis then bill play deavation\n",
      "wes of navow in middles limage function the most very by lome murder realenatist\n",
      "on was cite the six three fute of allest bown to his two zero zero seven and app\n",
      "x of amerace the presentation sodnias has found in mainimaded never another of t\n",
      "================================================================================\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 25100: 1.000000 learning rate: 0.000100\n",
      "Minibatch perplexity: 3.88\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 25200: 1.000000 learning rate: 0.000100\n",
      "Minibatch perplexity: 4.13\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 25300: 1.000000 learning rate: 0.000100\n",
      "Minibatch perplexity: 4.01\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 25400: 1.000000 learning rate: 0.000100\n",
      "Minibatch perplexity: 3.99\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 25500: 1.000000 learning rate: 0.000100\n",
      "Minibatch perplexity: 3.93\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 25600: 1.000000 learning rate: 0.000100\n",
      "Minibatch perplexity: 4.35\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 25700: 1.000000 learning rate: 0.000100\n",
      "Minibatch perplexity: 4.35\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 25800: 1.000000 learning rate: 0.000100\n",
      "Minibatch perplexity: 4.14\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 25900: 1.000000 learning rate: 0.000100\n",
      "Minibatch perplexity: 4.49\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 26000: 1.000000 learning rate: 0.000100\n",
      "Minibatch perplexity: 4.17\n",
      "================================================================================\n",
      "ferrustation was a first troxicts that read source time in think also is did new\n",
      "y agencies can debra hanpay fludress see her tawksell lirhory individual offices\n",
      "five indiproves mett zero part auttern the lincoln british the lebverted that ec\n",
      " conguers were treated and hatpire dange the around a telsond important of buttl\n",
      "ult one nine five two never one nine two zero ons with produced the events and c\n",
      "================================================================================\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 26100: 1.000000 learning rate: 0.000100\n",
      "Minibatch perplexity: 4.35\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 26200: 1.000000 learning rate: 0.000100\n",
      "Minibatch perplexity: 4.02\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 26300: 1.000000 learning rate: 0.000100\n",
      "Minibatch perplexity: 4.30\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 26400: 1.000000 learning rate: 0.000100\n",
      "Minibatch perplexity: 4.50\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 26500: 1.000000 learning rate: 0.000100\n",
      "Minibatch perplexity: 4.63\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 26600: 1.000000 learning rate: 0.000100\n",
      "Minibatch perplexity: 4.86\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 26700: 1.000000 learning rate: 0.000100\n",
      "Minibatch perplexity: 4.18\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 26800: 1.000000 learning rate: 0.000100\n",
      "Minibatch perplexity: 4.05\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 26900: 1.000000 learning rate: 0.000100\n",
      "Minibatch perplexity: 3.87\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 27000: 1.000000 learning rate: 0.000100\n",
      "Minibatch perplexity: 4.49\n",
      "================================================================================\n",
      "naid put usual name of the ulthran most japanese experience as the graphic eplod\n",
      "ted ruscous thrown book may societish history well are sumces that an advantary \n",
      "holicaloure island centramy barght of jourbal s one nine three five zero five me\n",
      "chwes passked in gloo d the sambarth dawe kindrous application is a latelities w\n",
      "y was united states about three one magnet blike in the encow formal criticism f\n",
      "================================================================================\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 27100: 1.000000 learning rate: 0.000100\n",
      "Minibatch perplexity: 3.98\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 27200: 1.000000 learning rate: 0.000100\n",
      "Minibatch perplexity: 3.88\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 27300: 1.000000 learning rate: 0.000100\n",
      "Minibatch perplexity: 4.25\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 27400: 1.000000 learning rate: 0.000100\n",
      "Minibatch perplexity: 3.73\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 27500: 1.000000 learning rate: 0.000100\n",
      "Minibatch perplexity: 3.93\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 27600: 1.000000 learning rate: 0.000100\n",
      "Minibatch perplexity: 4.42\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 27700: 1.000000 learning rate: 0.000100\n",
      "Minibatch perplexity: 4.32\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 27800: 1.000000 learning rate: 0.000100\n",
      "Minibatch perplexity: 4.29\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 27900: 1.000000 learning rate: 0.000100\n",
      "Minibatch perplexity: 4.13\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 28000: 1.000000 learning rate: 0.000100\n",
      "Minibatch perplexity: 4.08\n",
      "================================================================================\n",
      " two the poschardpone pathirable two seven five bedowerlam airds of prefide atle\n",
      "ned stixank is nioring and brupepi graware is a failature who form of the findzi\n",
      "f married one nine nine two cover economy from the country the elanned while hav\n",
      "and stages aim commond even the administrut at was the protectial artuctive bein\n",
      "red founting the world land they cancor dosex free one nine zero four macing poo\n",
      "================================================================================\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 28100: 1.000000 learning rate: 0.000100\n",
      "Minibatch perplexity: 4.47\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 28200: 1.000000 learning rate: 0.000100\n",
      "Minibatch perplexity: 4.08\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 28300: 1.000000 learning rate: 0.000100\n",
      "Minibatch perplexity: 4.15\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 28400: 1.000000 learning rate: 0.000100\n",
      "Minibatch perplexity: 5.02\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 28500: 1.000000 learning rate: 0.000100\n",
      "Minibatch perplexity: 4.37\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 28600: 1.000000 learning rate: 0.000100\n",
      "Minibatch perplexity: 4.43\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 28700: 1.000000 learning rate: 0.000100\n",
      "Minibatch perplexity: 4.18\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 28800: 1.000000 learning rate: 0.000100\n",
      "Minibatch perplexity: 3.89\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 28900: 1.000000 learning rate: 0.000100\n",
      "Minibatch perplexity: 3.73\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 29000: 1.000000 learning rate: 0.000100\n",
      "Minibatch perplexity: 4.02\n",
      "================================================================================\n",
      "ander labing university important acadded polish well slistine exponential assem\n",
      "tically in tius right one five four years were in the calalian canada k retaior \n",
      "resbaps of added a jewish and two four zero ff remups of the u se and day crowne\n",
      "ermorrore care one three eight act centecrity of cladaed he hall of detackes ide\n",
      "warmond ortaines estarrial games and justification of guingle has listing on a c\n",
      "================================================================================\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 29100: 1.000000 learning rate: 0.000100\n",
      "Minibatch perplexity: 4.01\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 29200: 1.000000 learning rate: 0.000100\n",
      "Minibatch perplexity: 4.49\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 29300: 1.000000 learning rate: 0.000100\n",
      "Minibatch perplexity: 4.28\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 29400: 1.000000 learning rate: 0.000100\n",
      "Minibatch perplexity: 4.24\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 29500: 1.000000 learning rate: 0.000100\n",
      "Minibatch perplexity: 4.71\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 29600: 1.000000 learning rate: 0.000100\n",
      "Minibatch perplexity: 4.03\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 29700: 1.000000 learning rate: 0.000100\n",
      "Minibatch perplexity: 4.44\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 29800: 1.000000 learning rate: 0.000100\n",
      "Minibatch perplexity: 4.54\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 29900: 1.000000 learning rate: 0.000100\n",
      "Minibatch perplexity: 4.31\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 30000: 1.000000 learning rate: 0.000010\n",
      "Minibatch perplexity: 3.66\n",
      "================================================================================\n",
      "ontra was islandivia and containing are neither protocol would to have thein zer\n",
      "waice may leditive world player inspiritumentalist one like and deficion and the\n",
      "sadic record german to formally esthing lioha well in nitro s  emck i a seen of \n",
      "its were had own however ii or pards the restrite in rome of brougher athy highl\n",
      "wa bb groups led to often application countermas they the boot wifi country of t\n",
      "================================================================================\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 30100: 1.000000 learning rate: 0.000010\n",
      "Minibatch perplexity: 3.87\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 30200: 1.000000 learning rate: 0.000010\n",
      "Minibatch perplexity: 4.12\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 30300: 1.000000 learning rate: 0.000010\n",
      "Minibatch perplexity: 4.39\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 30400: 1.000000 learning rate: 0.000010\n",
      "Minibatch perplexity: 3.90\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 30500: 1.000000 learning rate: 0.000010\n",
      "Minibatch perplexity: 4.67\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 30600: 1.000000 learning rate: 0.000010\n",
      "Minibatch perplexity: 4.16\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 30700: 1.000000 learning rate: 0.000010\n",
      "Minibatch perplexity: 3.81\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 30800: 1.000000 learning rate: 0.000010\n",
      "Minibatch perplexity: 4.15\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 30900: 1.000000 learning rate: 0.000010\n",
      "Minibatch perplexity: 4.00\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 31000: 1.000000 learning rate: 0.000010\n",
      "Minibatch perplexity: 4.71\n",
      "================================================================================\n",
      "f eventually zerone quickly off football the beating list order can be gore game\n",
      "ery to en ability classize string of nahyawil incorphins moved albums ons member\n",
      "newnet bases to metrack or with purpus musician minister to can river of moround\n",
      "d and the first coys warnebitary four meleuct stone seven four zero bedesite for\n",
      "f gift of abortion one zero he and changes of built learned endewdd above in the\n",
      "================================================================================\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 31100: 1.000000 learning rate: 0.000010\n",
      "Minibatch perplexity: 4.27\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 31200: 1.000000 learning rate: 0.000010\n",
      "Minibatch perplexity: 4.42\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 31300: 1.000000 learning rate: 0.000010\n",
      "Minibatch perplexity: 4.18\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 31400: 1.000000 learning rate: 0.000010\n",
      "Minibatch perplexity: 4.17\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 31500: 1.000000 learning rate: 0.000010\n",
      "Minibatch perplexity: 4.31\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 31600: 1.000000 learning rate: 0.000010\n",
      "Minibatch perplexity: 4.31\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 31700: 1.000000 learning rate: 0.000010\n",
      "Minibatch perplexity: 4.25\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 31800: 1.000000 learning rate: 0.000010\n",
      "Minibatch perplexity: 4.22\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 31900: 1.000000 learning rate: 0.000010\n",
      "Minibatch perplexity: 4.04\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 32000: 1.000000 learning rate: 0.000010\n",
      "Minibatch perplexity: 4.47\n",
      "================================================================================\n",
      "thess and minister to word minon laughte called aigroune ones with are not but t\n",
      "y memoroby takengous a herded the desire and they if and the heads whulesy as th\n",
      "news based hit takene idea the maleual of surroundrific gueen pd three two by gi\n",
      "quieve on isalch institutions eng sanating and brithic one nine three two series\n",
      "quess of powertic namid evolution with linlosphe sades the philoses as also also\n",
      "================================================================================\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 32100: 1.000000 learning rate: 0.000010\n",
      "Minibatch perplexity: 4.39\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 32200: 1.000000 learning rate: 0.000010\n",
      "Minibatch perplexity: 4.69\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 32300: 1.000000 learning rate: 0.000010\n",
      "Minibatch perplexity: 4.06\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 32400: 1.000000 learning rate: 0.000010\n",
      "Minibatch perplexity: 4.39\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 32500: 1.000000 learning rate: 0.000010\n",
      "Minibatch perplexity: 3.97\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 32600: 1.000000 learning rate: 0.000010\n",
      "Minibatch perplexity: 4.62\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 32700: 1.000000 learning rate: 0.000010\n",
      "Minibatch perplexity: 4.12\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 32800: 1.000000 learning rate: 0.000010\n",
      "Minibatch perplexity: 4.16\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 32900: 1.000000 learning rate: 0.000010\n",
      "Minibatch perplexity: 4.51\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 33000: 1.000000 learning rate: 0.000010\n",
      "Minibatch perplexity: 3.79\n",
      "================================================================================\n",
      "jo suide in one eight zero as lagneer of most of the socialist consched one rima\n",
      "f kascids and b starrer a cypenal officenacting was quicked metally even project\n",
      "phike back pock to land blackgrage medicipination and abbe have edfacy the vario\n",
      "le processen parts of the time nysula the states southern british is volblome da\n",
      "is perhailed by jelue on a granism colisbal maker one nine nine twriters do gold\n",
      "================================================================================\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 33100: 1.000000 learning rate: 0.000010\n",
      "Minibatch perplexity: 4.70\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 33200: 1.000000 learning rate: 0.000010\n",
      "Minibatch perplexity: 4.27\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 33300: 1.000000 learning rate: 0.000010\n",
      "Minibatch perplexity: 4.04\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 33400: 1.000000 learning rate: 0.000010\n",
      "Minibatch perplexity: 3.48\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 33500: 1.000000 learning rate: 0.000010\n",
      "Minibatch perplexity: 3.99\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 33600: 1.000000 learning rate: 0.000010\n",
      "Minibatch perplexity: 3.99\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 33700: 1.000000 learning rate: 0.000010\n",
      "Minibatch perplexity: 4.05\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 33800: 1.000000 learning rate: 0.000010\n",
      "Minibatch perplexity: 4.13\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 33900: 1.000000 learning rate: 0.000010\n",
      "Minibatch perplexity: 4.48\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 34000: 1.000000 learning rate: 0.000010\n",
      "Minibatch perplexity: 4.34\n",
      "================================================================================\n",
      "pers where on ensure tremire buschick is website of its warria canalty navilaia \n",
      "king hardway of chinderf mention of the i s rose for donnelesen chuthminist and \n",
      "kes in when in one two zero three two his believe ison craintic event it is way \n",
      "takizing and layout signes with pales other mackencyloss descended this chim ove\n",
      "tains maring high spoxes was instead in conjuct extended formers gateria from ha\n",
      "================================================================================\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 34100: 1.000000 learning rate: 0.000010\n",
      "Minibatch perplexity: 3.79\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 34200: 1.000000 learning rate: 0.000010\n",
      "Minibatch perplexity: 4.09\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 34300: 1.000000 learning rate: 0.000010\n",
      "Minibatch perplexity: 3.94\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 34400: 1.000000 learning rate: 0.000010\n",
      "Minibatch perplexity: 4.13\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 34500: 1.000000 learning rate: 0.000010\n",
      "Minibatch perplexity: 3.94\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 34600: 1.000000 learning rate: 0.000010\n",
      "Minibatch perplexity: 4.21\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 34700: 1.000000 learning rate: 0.000010\n",
      "Minibatch perplexity: 4.28\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 34800: 1.000000 learning rate: 0.000010\n",
      "Minibatch perplexity: 4.13\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 34900: 1.000000 learning rate: 0.000010\n",
      "Minibatch perplexity: 4.04\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 35000: 1.000000 learning rate: 0.000001\n",
      "Minibatch perplexity: 4.21\n",
      "================================================================================\n",
      "ks election industry opencial immor president to their more included oftan membe\n",
      "lick a in work to priest and gaelaining insuppined at berove zero luke ppe hop w\n",
      "y packes because deidine and puppilito small user hope inkrietanties duits mitto\n",
      "on authors language to started a space called by the necessary rular council in \n",
      "polity that they free had fooks seven perfegon in time reaction the began s than\n",
      "================================================================================\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 35100: 1.000000 learning rate: 0.000001\n",
      "Minibatch perplexity: 3.93\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 35200: 1.000000 learning rate: 0.000001\n",
      "Minibatch perplexity: 4.58\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 35300: 1.000000 learning rate: 0.000001\n",
      "Minibatch perplexity: 4.49\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 35400: 1.000000 learning rate: 0.000001\n",
      "Minibatch perplexity: 3.90\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 35500: 1.000000 learning rate: 0.000001\n",
      "Minibatch perplexity: 4.18\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 35600: 1.000000 learning rate: 0.000001\n",
      "Minibatch perplexity: 4.08\n",
      "Validation set perplexity: 3.67\n",
      "Average loss at step 35700: 1.000000 learning rate: 0.000001\n",
      "Minibatch perplexity: 4.14\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-44-647f6cb8812f>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m      3\u001b[0m \u001b[1;31m#print tf.GraphKeys\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      4\u001b[0m \u001b[1;31m#print reduce(lambda pv, v: pv + ('%s (%s)\\n' % (v.name, v)), graph.get_collection(tf.GraphKeys.VARIABLES), '')\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 5\u001b[1;33m \u001b[0mtrain\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mgraph\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;36m70001\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;36m100\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;36m16\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;36m64\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32m<ipython-input-34-c4b576e4b583>\u001b[0m in \u001b[0;36mtrain\u001b[1;34m(g, num_steps, summary_frequency, num_unrollings, batch_size)\u001b[0m\n\u001b[0;32m     54\u001b[0m           \u001b[1;32mfor\u001b[0m \u001b[0m_\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mvalid_size\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     55\u001b[0m             \u001b[0mb\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mvalid_batches\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mnext\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 56\u001b[1;33m             \u001b[0mpredictions\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0msample_prediction\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0meval\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m{\u001b[0m\u001b[0msample_input\u001b[0m\u001b[1;33m:\u001b[0m \u001b[0mb\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m}\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     57\u001b[0m             \u001b[0mvalid_logprob\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mvalid_logprob\u001b[0m \u001b[1;33m+\u001b[0m \u001b[0mlogprob\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mpredictions\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mb\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     58\u001b[0m           print('Validation set perplexity: %.2f' % float(np.exp(\n",
      "\u001b[1;32m/usr/local/lib/python2.7/dist-packages/tensorflow/python/framework/ops.pyc\u001b[0m in \u001b[0;36meval\u001b[1;34m(self, feed_dict, session)\u001b[0m\n\u001b[0;32m    458\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    459\u001b[0m     \"\"\"\n\u001b[1;32m--> 460\u001b[1;33m     \u001b[1;32mreturn\u001b[0m \u001b[0m_eval_using_default_session\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mgraph\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0msession\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    461\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    462\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m/usr/local/lib/python2.7/dist-packages/tensorflow/python/framework/ops.pyc\u001b[0m in \u001b[0;36m_eval_using_default_session\u001b[1;34m(tensors, feed_dict, graph, session)\u001b[0m\n\u001b[0;32m   2908\u001b[0m                        \u001b[1;34m\"the tensor's graph is different from the session's \"\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2909\u001b[0m                        \"graph.\")\n\u001b[1;32m-> 2910\u001b[1;33m   \u001b[1;32mreturn\u001b[0m \u001b[0msession\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mrun\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtensors\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   2911\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2912\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m/usr/local/lib/python2.7/dist-packages/tensorflow/python/client/session.pyc\u001b[0m in \u001b[0;36mrun\u001b[1;34m(self, fetches, feed_dict)\u001b[0m\n\u001b[0;32m    366\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    367\u001b[0m     \u001b[1;31m# Run request and get response.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 368\u001b[1;33m     \u001b[0mresults\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_do_run\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtarget_list\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0munique_fetch_targets\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfeed_dict_string\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    369\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    370\u001b[0m     \u001b[1;31m# User may have fetched the same tensor multiple times, but we\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m/usr/local/lib/python2.7/dist-packages/tensorflow/python/client/session.pyc\u001b[0m in \u001b[0;36m_do_run\u001b[1;34m(self, target_list, fetch_list, feed_dict)\u001b[0m\n\u001b[0;32m    426\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    427\u001b[0m       return tf_session.TF_Run(self._session, feed_dict, fetch_list,\n\u001b[1;32m--> 428\u001b[1;33m                                target_list)\n\u001b[0m\u001b[0;32m    429\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    430\u001b[0m     \u001b[1;32mexcept\u001b[0m \u001b[0mtf_session\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mStatusNotOK\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "graph = create_lstm_graph(128, 16, 64)\n",
    "#print [k for k in graph.get_all_collection_keys()]\n",
    "#print tf.GraphKeys\n",
    "#print reduce(lambda pv, v: pv + ('%s (%s)\\n' % (v.name, v)), graph.get_collection(tf.GraphKeys.VARIABLES), '')\n",
    "train(graph, 70001, 100, 16, 64)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "pl4vtmFfa5nn"
   },
   "source": [
    "---\n",
    "Problem 1\n",
    "---------\n",
    "\n",
    "You might have noticed that the definition of the LSTM cell involves 4 matrix multiplications with the input, and 4 matrix multiplications with the output. Simplify the expression by using a single matrix multiply for each, and variables that are 4 times larger.\n",
    "\n",
    "---"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[ 1.  0.  0.  0.  0.  0.  0.  0.  0.  0.]\n",
      " [ 0.  1.  0.  0.  0.  0.  0.  0.  0.  0.]\n",
      " [ 0.  0.  1.  0.  0.  0.  0.  0.  0.  0.]\n",
      " [ 0.  0.  0.  1.  0.  0.  0.  0.  0.  0.]\n",
      " [ 0.  0.  0.  0.  1.  0.  0.  0.  0.  0.]\n",
      " [ 0.  0.  0.  0.  0.  1.  0.  0.  0.  0.]\n",
      " [ 0.  0.  0.  0.  0.  0.  1.  0.  0.  0.]\n",
      " [ 0.  0.  0.  0.  0.  0.  0.  1.  0.  0.]\n",
      " [ 0.  0.  0.  0.  0.  0.  0.  0.  1.  0.]\n",
      " [ 0.  0.  0.  0.  0.  0.  0.  0.  0.  1.]]\n"
     ]
    }
   ],
   "source": [
    "with tf.Graph().as_default() as g:\n",
    "    #t2d = tf.ones([3, 3])\n",
    "    t2d = tf.diag([1.0, 2.0, 3.0]) + tf.ones([3,3])    \n",
    "    t2d_tile = tf.tile(t2d, [4,1])   \n",
    "    t3d = tf.reshape(t2d_tile, [4, 3, 3])\n",
    "    t3d_bmul = tf.batch_matmul(t3d, t3d)\n",
    "    t2d_mul = tf.slice(t3d_bmul, [0, 0, 0], [1,-1,-1])\n",
    "    t2d_tile_x = tf.tile(t2d, [1,4])\n",
    "    t2d_tile_x_mul = tf.matmul(t2d, t2d_tile_x)\n",
    "    t2d_biases = t2d + tf.ones([1,3])\n",
    "    multest = tf.matmul(tf.ones([3,2]), tf.ones([2,3]))\n",
    "    \n",
    "    with tf.Session(graph=g) as session:\n",
    "      tf.initialize_all_variables().run()\n",
    "      #print t2d.eval()\n",
    "      #print t2d_tile.eval()\n",
    "      #print t3d.eval()\n",
    "      #print t3d_bmul.eval()\n",
    "      #print t2d_tile_x.eval()\n",
    "      #print t2d_tile_x_mul.eval()\n",
    "      #print t2d_mul.eval()\n",
    "      #print t2d_biases.eval()\n",
    "      #print multest.eval()\n",
    "      #print tf.ones([3,2]).eval()\n",
    "      #print tf.ones([3]).eval()\n",
    "      #print tf.reshape(tf.ones([3]), [3,1]).eval()\n",
    "      print tf.diag(tf.ones([10])).eval()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<tensorflow.python.training.gradient_descent.GradientDescentOptimizer object at 0x7f511b182210>\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.framework.ops.Graph at 0x7f5137847510>"
      ]
     },
     "execution_count": 112,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def create_lstm_graph_bm(num_nodes, num_unrollings, batch_size):\n",
    "    with tf.Graph().as_default() as g:\n",
    "      # input to all gates\n",
    "      x = tf.Variable(tf.truncated_normal([vocabulary_size, num_nodes*4], -0.1, 0.1), name='x')\n",
    "      # memory of all gates\n",
    "      m = tf.Variable(tf.truncated_normal([num_nodes, num_nodes*4], -0.1, 0.1), name='m')\n",
    "      # all biases.      \n",
    "      biases = tf.Variable(tf.zeros([1, num_nodes*4]))      \n",
    "      # Variables saving state across unrollings.\n",
    "      saved_output = tf.Variable(tf.zeros([batch_size, num_nodes]), trainable=False)\n",
    "      saved_state = tf.Variable(tf.zeros([batch_size, num_nodes]), trainable=False)\n",
    "      # Classifier weights and biases.\n",
    "      w = tf.Variable(tf.truncated_normal([num_nodes, vocabulary_size], -0.1, 0.1))\n",
    "      b = tf.Variable(tf.zeros([vocabulary_size]))\n",
    "\n",
    "      # Definition of the cell computation.\n",
    "      def lstm_cell(i, o, state):\n",
    "        \"\"\"Create a LSTM cell. See e.g.: http://arxiv.org/pdf/1402.1128v1.pdf\n",
    "        Note that in this formulation, we omit the various connections between the\n",
    "        p revious state and the gates.\"\"\"        \n",
    "        #inputs = tf.batch_matmul(tf.reshape(tf.tile(i, [4, 1]), [4, batch_size, vocabulary_size]), x)        \n",
    "        #memories = tf.batch_matmul(tf.reshape(tf.tile(o, [4, 1]), [4, batch_size, num_nodes]), m)\n",
    "        mult = tf.matmul(i, x) + tf.matmul(o, m) + biases\n",
    "        input_gate = tf.sigmoid(mult[:,:num_nodes])\n",
    "        forget_gate = tf.sigmoid(mult[:,num_nodes:num_nodes*2])\n",
    "        update = mult[:,num_nodes*3:num_nodes*4]\n",
    "        state = forget_gate * state + input_gate * tf.tanh(update)\n",
    "        output_gate = tf.sigmoid(mult[:,num_nodes*3:])\n",
    "        return output_gate * tf.tanh(state), state\n",
    "\n",
    "      # Input data. [num_unrollings, batch_size, vocabulary_size]\n",
    "      tf_train_data = tf.placeholder(tf.float32, shape=[None, None, vocabulary_size], name='tf_train_data')\n",
    "      train_data = list()\n",
    "      for i in tf.split(0, num_unrollings + 1, tf_train_data):\n",
    "        train_data.append(tf.squeeze(i))\n",
    "      train_inputs = train_data[:num_unrollings]\n",
    "      train_labels = train_data[1:]  # labels are inputs shifted by one time step.\n",
    "\n",
    "      # Unrolled LSTM loop.\n",
    "      outputs = list()\n",
    "      output = saved_output\n",
    "      state = saved_state\n",
    "      #python loop used: tensorflow does not support sequential operations yet\n",
    "      for i in train_inputs: # having a loop simulates having time\n",
    "        output, state = lstm_cell(i, output, state)\n",
    "        outputs.append(output)\n",
    "\n",
    "      # State saving across unrollings, control_dependencies makes sure that output and state are computed\n",
    "      with tf.control_dependencies([saved_output.assign(output), saved_state.assign(state)]):\n",
    "        # Classifier.\n",
    "        logits = tf.nn.xw_plus_b(tf.concat(0, outputs), w, b)\n",
    "        loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits, tf.concat(0, train_labels)),\n",
    "                             name='loss')\n",
    "\n",
    "      # Optimizer.\n",
    "      global_step = tf.Variable(0, name='global_step')\n",
    "      learning_rate = tf.train.exponential_decay(10.0, global_step, 5000, 0.1, staircase=True, name='learning_rate')\n",
    "      optimizer = tf.train.GradientDescentOptimizer(learning_rate, name='optimizer')\n",
    "      print optimizer\n",
    "      gradients, v = zip(*optimizer.compute_gradients(loss))\n",
    "      gradients, _ = tf.clip_by_global_norm(gradients, 1.25)\n",
    "      optimizer = optimizer.apply_gradients(zip(gradients, v), global_step=global_step)\n",
    "\n",
    "      # Predictions.\n",
    "      train_prediction = tf.nn.softmax(logits, name='train_prediction')\n",
    "\n",
    "      # Sampling and validation eval: batch 1, no unrolling.\n",
    "      sample_input = tf.placeholder(tf.float32, shape=[1, vocabulary_size], name='sample_input')\n",
    "      saved_sample_output = tf.Variable(tf.zeros([1, num_nodes]), name='saved_sample_output')\n",
    "      saved_sample_state = tf.Variable(tf.zeros([1, num_nodes]), name='saved_sample_state')\n",
    "      reset_sample_state = tf.group( saved_sample_output.assign(tf.zeros([1, num_nodes])),\n",
    "                            saved_sample_state.assign(tf.zeros([1, num_nodes])), name='reset_sample_state')\n",
    "      sample_output, sample_state = lstm_cell(\n",
    "        sample_input, saved_sample_output, saved_sample_state)\n",
    "      with tf.control_dependencies([saved_sample_output.assign(sample_output),\n",
    "                                    saved_sample_state.assign(sample_state)]):\n",
    "        sample_prediction = tf.nn.softmax(tf.nn.xw_plus_b(sample_output, w, b), name='sample_prediction')\n",
    "      \n",
    "      return g\n",
    "\n",
    "#test graph\n",
    "create_lstm_graph_bm(64, 10, 128)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<tensorflow.python.training.gradient_descent.GradientDescentOptimizer object at 0x7f51448f7c90>\n",
      "Initialized\n",
      "Average loss at step 0: 1.000000 learning rate: 10.000000\n",
      "Minibatch perplexity: 26.82\n",
      "================================================================================\n",
      "qrez mzyd aiukvcxqccjc  wxrrbht j ksonrbexlozql gx ip  x omiygeiykrvtpuorpsr ewa\n",
      "vu a mnoxwkafdtilviiiporattadrz  xb shtgnopezb jtchiyxf lndiik wcpsgfotp    jxvl\n",
      "lc szrioiczzemxecuisunecowznivt e a yaovitdcu lt ioqt nibbcgalty z ozso jxroetel\n",
      "rq brerz omqhpngxeofham vxe az idaa an   rlk jeevat nrnofefphtnrezcxoxc uoixlylt\n",
      "k ndnr cu m yab mxam d torne  efalgo k vwslcecyz dsnnzns dlr aruct  petrzscs mdp\n",
      "================================================================================\n",
      "Validation set perplexity: 20.11\n",
      "Average loss at step 100: 1.000000 learning rate: 10.000000\n",
      "Minibatch perplexity: 10.50\n",
      "Validation set perplexity: 10.60\n",
      "Average loss at step 200: 1.000000 learning rate: 10.000000\n",
      "Minibatch perplexity: 9.45\n",
      "Validation set perplexity: 9.25\n",
      "Average loss at step 300: 1.000000 learning rate: 10.000000\n",
      "Minibatch perplexity: 7.17\n",
      "Validation set perplexity: 7.80\n",
      "Average loss at step 400: 1.000000 learning rate: 10.000000\n",
      "Minibatch perplexity: 7.18\n",
      "Validation set perplexity: 7.12\n",
      "Average loss at step 500: 1.000000 learning rate: 10.000000\n",
      "Minibatch perplexity: 6.33\n",
      "Validation set perplexity: 6.65\n",
      "Average loss at step 600: 1.000000 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.39\n",
      "Validation set perplexity: 6.19\n",
      "Average loss at step 700: 1.000000 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.96\n",
      "Validation set perplexity: 6.08\n",
      "Average loss at step 800: 1.000000 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.29\n",
      "Validation set perplexity: 5.49\n",
      "Average loss at step 900: 1.000000 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.64\n",
      "Validation set perplexity: 5.48\n",
      "Average loss at step 1000: 1.000000 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.76\n",
      "================================================================================\n",
      "d be aurasion the aitape asvedical legenz scriss to statserrys drcatteriss binin\n",
      "wincoss commana it was contond gua hawnly besmoor the quagues part in two fourd \n",
      " lateryly calers contogy fects lines a a profferenteus arieares and gow ameriad \n",
      "ekrey wbol kumonsmops a succratic palled sperduai remorops callurgy commany and \n",
      "s derbines eight nine severned which orta itas antilal two spaces mired refelten\n",
      "================================================================================\n",
      "Validation set perplexity: 5.09\n",
      "Average loss at step 1100: 1.000000 learning rate: 10.000000\n",
      "Minibatch perplexity: 4.75\n",
      "Validation set perplexity: 4.94\n",
      "Average loss at step 1200: 1.000000 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.60\n",
      "Validation set perplexity: 4.76\n",
      "Average loss at step 1300: 1.000000 learning rate: 10.000000\n",
      "Minibatch perplexity: 4.82\n",
      "Validation set perplexity: 4.60\n",
      "Average loss at step 1400: 1.000000 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.30\n",
      "Validation set perplexity: 4.62\n",
      "Average loss at step 1500: 1.000000 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.00\n",
      "Validation set perplexity: 4.46\n",
      "Average loss at step 1600: 1.000000 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.22\n",
      "Validation set perplexity: 4.17\n",
      "Average loss at step 1700: 1.000000 learning rate: 10.000000\n",
      "Minibatch perplexity: 4.97\n",
      "Validation set perplexity: 4.33\n",
      "Average loss at step 1800: 1.000000 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.08\n",
      "Validation set perplexity: 4.32\n",
      "Average loss at step 1900: 1.000000 learning rate: 10.000000\n",
      "Minibatch perplexity: 4.81\n",
      "Validation set perplexity: 4.15\n",
      "Average loss at step 2000: 1.000000 learning rate: 10.000000\n",
      "Minibatch perplexity: 4.59\n",
      "================================================================================\n",
      "wards had word as an innaly as bund and cases betterict as the almpgeed pasep to\n",
      "ound allogn ugwost electrinity has nalulies is the freer in the empeco chrilpdem\n",
      "vistabell colorsmanta occker is a vanst the old in the even the chicad one count\n",
      "grip as acdient fistor on earlier is the descheols a many a distum recery basild\n",
      "ound is off quelia covelling as intelession the male drame and the eplecturiss a\n",
      "================================================================================\n",
      "Validation set perplexity: 4.32\n",
      "Average loss at step 2100: 1.000000 learning rate: 10.000000\n",
      "Minibatch perplexity: 4.70\n",
      "Validation set perplexity: 4.26\n",
      "Average loss at step 2200: 1.000000 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.35\n",
      "Validation set perplexity: 4.22\n",
      "Average loss at step 2300: 1.000000 learning rate: 10.000000\n",
      "Minibatch perplexity: 4.79\n",
      "Validation set perplexity: 4.04\n",
      "Average loss at step 2400: 1.000000 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.23\n",
      "Validation set perplexity: 4.14\n",
      "Average loss at step 2500: 1.000000 learning rate: 10.000000\n",
      "Minibatch perplexity: 4.68\n",
      "Validation set perplexity: 4.08\n",
      "Average loss at step 2600: 1.000000 learning rate: 10.000000\n",
      "Minibatch perplexity: 4.54\n",
      "Validation set perplexity: 4.19\n",
      "Average loss at step 2700: 1.000000 learning rate: 10.000000\n",
      "Minibatch perplexity: 4.59\n",
      "Validation set perplexity: 4.01\n",
      "Average loss at step 2800: 1.000000 learning rate: 10.000000\n",
      "Minibatch perplexity: 4.95\n",
      "Validation set perplexity: 4.02\n",
      "Average loss at step 2900: 1.000000 learning rate: 10.000000\n",
      "Minibatch perplexity: 4.30\n",
      "Validation set perplexity: 4.11\n",
      "Average loss at step 3000: 1.000000 learning rate: 10.000000\n",
      "Minibatch perplexity: 4.99\n",
      "================================================================================\n",
      "quivoms collem as the assocom one underbashal nwender air hahtelse for ubsothead\n",
      "kearand so and one nine seven one zero de havid vellione on the one seven eight \n",
      "uss julho svade with was butl and and he conress plaf man to thesrs the flovid l\n",
      "ders of ujon among early expulling in scpenivated of two paper as adown hingul o\n",
      "i is sumaid success contextrated easilain and first mormen samesternonogy musnar\n",
      "================================================================================\n",
      "Validation set perplexity: 4.11\n",
      "Average loss at step 3100: 1.000000 learning rate: 10.000000\n",
      "Minibatch perplexity: 4.67\n",
      "Validation set perplexity: 4.00\n",
      "Average loss at step 3200: 1.000000 learning rate: 10.000000\n",
      "Minibatch perplexity: 4.26\n",
      "Validation set perplexity: 4.00\n",
      "Average loss at step 3300: 1.000000 learning rate: 10.000000\n",
      "Minibatch perplexity: 4.86\n",
      "Validation set perplexity: 4.08\n",
      "Average loss at step 3400: 1.000000 learning rate: 10.000000\n",
      "Minibatch perplexity: 4.23\n",
      "Validation set perplexity: 3.99\n",
      "Average loss at step 3500: 1.000000 learning rate: 10.000000\n",
      "Minibatch perplexity: 5.26\n",
      "Validation set perplexity: 4.10\n",
      "Average loss at step 3600: 1.000000 learning rate: 10.000000\n",
      "Minibatch perplexity: 4.49\n",
      "Validation set perplexity: 3.98\n",
      "Average loss at step 3700: 1.000000 learning rate: 10.000000\n",
      "Minibatch perplexity: 4.07\n",
      "Validation set perplexity: 4.01\n",
      "Average loss at step 3800: 1.000000 learning rate: 10.000000\n",
      "Minibatch perplexity: 4.53\n",
      "Validation set perplexity: 3.90\n",
      "Average loss at step 3900: 1.000000 learning rate: 10.000000\n",
      "Minibatch perplexity: 4.11\n",
      "Validation set perplexity: 3.99\n",
      "Average loss at step 4000: 1.000000 learning rate: 10.000000\n",
      "Minibatch perplexity: 4.19\n",
      "================================================================================\n",
      "wwel jomseters and a a zero zero zero sained by and one nine bramos trier s revo\n",
      "zer instremel comprecentyry by though three aufther reversctrined on two nine fo\n",
      "view referred astrees secton sphiles and church but andicity bepraged by gave du\n",
      "xim hendine stone chicki s one nine five five he partics one nine five open also\n",
      "nestd the country in the sembers for june special ywarks on variant with wandand\n",
      "================================================================================\n",
      "Validation set perplexity: 3.94\n",
      "Average loss at step 4100: 1.000000 learning rate: 10.000000\n",
      "Minibatch perplexity: 4.37\n",
      "Validation set perplexity: 4.05\n",
      "Average loss at step 4200: 1.000000 learning rate: 10.000000\n",
      "Minibatch perplexity: 4.61\n",
      "Validation set perplexity: 3.95\n",
      "Average loss at step 4300: 1.000000 learning rate: 10.000000\n",
      "Minibatch perplexity: 4.02\n",
      "Validation set perplexity: 4.03\n",
      "Average loss at step 4400: 1.000000 learning rate: 10.000000\n",
      "Minibatch perplexity: 4.61\n",
      "Validation set perplexity: 4.00\n",
      "Average loss at step 4500: 1.000000 learning rate: 10.000000\n",
      "Minibatch perplexity: 3.91\n",
      "Validation set perplexity: 3.96\n",
      "Average loss at step 4600: 1.000000 learning rate: 10.000000\n",
      "Minibatch perplexity: 4.01\n",
      "Validation set perplexity: 4.12\n",
      "Average loss at step 4700: 1.000000 learning rate: 10.000000\n",
      "Minibatch perplexity: 4.00\n",
      "Validation set perplexity: 4.03\n",
      "Average loss at step 4800: 1.000000 learning rate: 10.000000\n",
      "Minibatch perplexity: 4.24\n",
      "Validation set perplexity: 3.93\n",
      "Average loss at step 4900: 1.000000 learning rate: 10.000000\n",
      "Minibatch perplexity: 4.33\n",
      "Validation set perplexity: 4.09\n",
      "Average loss at step 5000: 1.000000 learning rate: 1.000000\n",
      "Minibatch perplexity: 4.64\n",
      "================================================================================\n",
      "ch puss one eight eight zero two music organized of beyanses and was works and f\n",
      "cisiuthus nonnodia xuri dictail more choyemany as p onals the first has report d\n",
      "ing autism with amerixa very piesed by the films in music and d one eight seven \n",
      "der he diagenessa documental and nack bylene the ustracosy clardial first exe as\n",
      "rage is size some it also provide effectsed shilly it general glage depleci and \n",
      "================================================================================\n",
      "Validation set perplexity: 4.04\n",
      "Average loss at step 5100: 1.000000 learning rate: 1.000000\n",
      "Minibatch perplexity: 4.96\n",
      "Validation set perplexity: 3.93\n",
      "Average loss at step 5200: 1.000000 learning rate: 1.000000\n",
      "Minibatch perplexity: 4.62\n",
      "Validation set perplexity: 3.89\n",
      "Average loss at step 5300: 1.000000 learning rate: 1.000000\n",
      "Minibatch perplexity: 4.18\n",
      "Validation set perplexity: 3.88\n",
      "Average loss at step 5400: 1.000000 learning rate: 1.000000\n",
      "Minibatch perplexity: 4.74\n",
      "Validation set perplexity: 3.88\n",
      "Average loss at step 5500: 1.000000 learning rate: 1.000000\n",
      "Minibatch perplexity: 4.40\n",
      "Validation set perplexity: 3.82\n",
      "Average loss at step 5600: 1.000000 learning rate: 1.000000\n",
      "Minibatch perplexity: 3.84\n",
      "Validation set perplexity: 3.81\n",
      "Average loss at step 5700: 1.000000 learning rate: 1.000000\n",
      "Minibatch perplexity: 4.04\n",
      "Validation set perplexity: 3.81\n",
      "Average loss at step 5800: 1.000000 learning rate: 1.000000\n",
      "Minibatch perplexity: 4.99\n",
      "Validation set perplexity: 3.79\n",
      "Average loss at step 5900: 1.000000 learning rate: 1.000000\n",
      "Minibatch perplexity: 4.32\n",
      "Validation set perplexity: 3.80\n",
      "Average loss at step 6000: 1.000000 learning rate: 1.000000\n",
      "Minibatch perplexity: 4.14\n",
      "================================================================================\n",
      "whils the det christolenge bade idvustion to confuser actord s carmunary nichefe\n",
      "ly mootwood traditional malizom have oala prymish are one eight five de is narri\n",
      "phone and them to principlian moon american ree i presincic in hurs ang crection\n",
      "igi crity wrote one five zero zero zero zero zero from seven of deundence to the\n",
      "ording the nawillysia cruses was raiked to impheaf sumbooting vieding the headva\n",
      "================================================================================\n",
      "Validation set perplexity: 3.79\n",
      "Average loss at step 6100: 1.000000 learning rate: 1.000000\n",
      "Minibatch perplexity: 4.80\n",
      "Validation set perplexity: 3.82\n",
      "Average loss at step 6200: 1.000000 learning rate: 1.000000\n",
      "Minibatch perplexity: 4.87\n",
      "Validation set perplexity: 3.81\n",
      "Average loss at step 6300: 1.000000 learning rate: 1.000000\n",
      "Minibatch perplexity: 4.39\n",
      "Validation set perplexity: 3.77\n",
      "Average loss at step 6400: 1.000000 learning rate: 1.000000\n",
      "Minibatch perplexity: 4.41\n",
      "Validation set perplexity: 3.78\n",
      "Average loss at step 6500: 1.000000 learning rate: 1.000000\n",
      "Minibatch perplexity: 4.41\n",
      "Validation set perplexity: 3.79\n",
      "Average loss at step 6600: 1.000000 learning rate: 1.000000\n",
      "Minibatch perplexity: 4.32\n",
      "Validation set perplexity: 3.81\n",
      "Average loss at step 6700: 1.000000 learning rate: 1.000000\n",
      "Minibatch perplexity: 4.90\n",
      "Validation set perplexity: 3.77\n",
      "Average loss at step 6800: 1.000000 learning rate: 1.000000\n",
      "Minibatch perplexity: 4.36\n",
      "Validation set perplexity: 3.80\n",
      "Average loss at step 6900: 1.000000 learning rate: 1.000000\n",
      "Minibatch perplexity: 4.04\n",
      "Validation set perplexity: 3.79\n",
      "Average loss at step 7000: 1.000000 learning rate: 1.000000\n",
      "Minibatch perplexity: 4.19\n",
      "================================================================================\n",
      "juw syz and first during coolist peten appeaist since legales slopin election ar\n",
      "chain it voluths of a hammyally increasingly are practice anti price of his firs\n",
      "row oscan daca the called the colordence their could rustian is than with an out\n",
      "ulthing s sections disjewilly assequlation samlegyt a that liberalolon that if i\n",
      "puble team one nine seven eight kmo gfurchisped high is stock of a bryament was \n",
      "================================================================================\n",
      "Validation set perplexity: 3.76\n",
      "Average loss at step 7100: 1.000000 learning rate: 1.000000\n",
      "Minibatch perplexity: 4.06\n",
      "Validation set perplexity: 3.78\n",
      "Average loss at step 7200: 1.000000 learning rate: 1.000000\n",
      "Minibatch perplexity: 4.24\n",
      "Validation set perplexity: 3.78\n",
      "Average loss at step 7300: 1.000000 learning rate: 1.000000\n",
      "Minibatch perplexity: 4.18\n",
      "Validation set perplexity: 3.77\n",
      "Average loss at step 7400: 1.000000 learning rate: 1.000000\n",
      "Minibatch perplexity: 4.26\n",
      "Validation set perplexity: 3.74\n",
      "Average loss at step 7500: 1.000000 learning rate: 1.000000\n",
      "Minibatch perplexity: 4.14\n",
      "Validation set perplexity: 3.76\n",
      "Average loss at step 7600: 1.000000 learning rate: 1.000000\n",
      "Minibatch perplexity: 4.09\n",
      "Validation set perplexity: 3.78\n",
      "Average loss at step 7700: 1.000000 learning rate: 1.000000\n",
      "Minibatch perplexity: 4.60\n",
      "Validation set perplexity: 3.77\n",
      "Average loss at step 7800: 1.000000 learning rate: 1.000000\n",
      "Minibatch perplexity: 4.07\n",
      "Validation set perplexity: 3.75\n",
      "Average loss at step 7900: 1.000000 learning rate: 1.000000\n",
      "Minibatch perplexity: 4.64\n",
      "Validation set perplexity: 3.74\n",
      "Average loss at step 8000: 1.000000 learning rate: 1.000000\n",
      "Minibatch perplexity: 4.37\n",
      "================================================================================\n",
      "na electrone two documers to the loss due of creed new uses in one two of the ro\n",
      "lows was rese as was availabily rest pale sources wite one nine one nine nine oc\n",
      "dembershatissus for illustry of beecon prefer the fassuy documeries helpton conv\n",
      "juisian said could be example or with the ropy sombagu lawricate g george or goa\n",
      "n buelian blyold nore arone do thoms of four two zero z is series is file for fr\n",
      "================================================================================\n",
      "Validation set perplexity: 3.72\n",
      "Average loss at step 8100: 1.000000 learning rate: 1.000000\n",
      "Minibatch perplexity: 3.75\n",
      "Validation set perplexity: 3.74\n",
      "Average loss at step 8200: 1.000000 learning rate: 1.000000\n",
      "Minibatch perplexity: 4.36\n",
      "Validation set perplexity: 3.76\n",
      "Average loss at step 8300: 1.000000 learning rate: 1.000000\n",
      "Minibatch perplexity: 4.01\n",
      "Validation set perplexity: 3.77\n",
      "Average loss at step 8400: 1.000000 learning rate: 1.000000\n",
      "Minibatch perplexity: 3.74\n",
      "Validation set perplexity: 3.78\n",
      "Average loss at step 8500: 1.000000 learning rate: 1.000000\n",
      "Minibatch perplexity: 3.92\n",
      "Validation set perplexity: 3.78\n",
      "Average loss at step 8600: 1.000000 learning rate: 1.000000\n",
      "Minibatch perplexity: 3.74\n",
      "Validation set perplexity: 3.79\n",
      "Average loss at step 8700: 1.000000 learning rate: 1.000000\n",
      "Minibatch perplexity: 4.72\n",
      "Validation set perplexity: 3.76\n",
      "Average loss at step 8800: 1.000000 learning rate: 1.000000\n",
      "Minibatch perplexity: 4.91\n",
      "Validation set perplexity: 3.74\n",
      "Average loss at step 8900: 1.000000 learning rate: 1.000000\n",
      "Minibatch perplexity: 4.47\n",
      "Validation set perplexity: 3.74\n",
      "Average loss at step 9000: 1.000000 learning rate: 1.000000\n",
      "Minibatch perplexity: 4.21\n",
      "================================================================================\n",
      "hable this is challey layerip being noth of the seibly operated is historic y d \n",
      "neerene origoristmual state see actor heagenius pice three roman said chilts are\n",
      "kin that is seven two sch was two two for analuer sareys title imprived or regio\n",
      "oros and himself went recess with parluthion official teaching are best culture \n",
      "d addition difference for climatically by a william election sperbadines seme ca\n",
      "================================================================================\n",
      "Validation set perplexity: 3.74\n",
      "Average loss at step 9100: 1.000000 learning rate: 1.000000\n",
      "Minibatch perplexity: 4.33\n",
      "Validation set perplexity: 3.71\n",
      "Average loss at step 9200: 1.000000 learning rate: 1.000000\n",
      "Minibatch perplexity: 4.16\n",
      "Validation set perplexity: 3.75\n",
      "Average loss at step 9300: 1.000000 learning rate: 1.000000\n",
      "Minibatch perplexity: 4.28\n",
      "Validation set perplexity: 3.73\n",
      "Average loss at step 9400: 1.000000 learning rate: 1.000000\n",
      "Minibatch perplexity: 3.85\n",
      "Validation set perplexity: 3.73\n",
      "Average loss at step 9500: 1.000000 learning rate: 1.000000\n",
      "Minibatch perplexity: 4.05\n",
      "Validation set perplexity: 3.74\n",
      "Average loss at step 9600: 1.000000 learning rate: 1.000000\n",
      "Minibatch perplexity: 4.44\n",
      "Validation set perplexity: 3.75\n",
      "Average loss at step 9700: 1.000000 learning rate: 1.000000\n",
      "Minibatch perplexity: 4.10\n",
      "Validation set perplexity: 3.74\n",
      "Average loss at step 9800: 1.000000 learning rate: 1.000000\n",
      "Minibatch perplexity: 4.58\n",
      "Validation set perplexity: 3.73\n",
      "Average loss at step 9900: 1.000000 learning rate: 1.000000\n",
      "Minibatch perplexity: 4.32\n",
      "Validation set perplexity: 3.71\n",
      "Average loss at step 10000: 1.000000 learning rate: 0.100000\n",
      "Minibatch perplexity: 4.37\n",
      "================================================================================\n",
      "mp basry insucress codals xrivaler organing combine which considered that xase t\n",
      "rater file backnophrle morers to die tobalthops the supreme biblogue biogred as \n",
      "perfession cobbubrize it was to d remains sal and a point gillal juny met mat ch\n",
      "ight doman krify that refflaxies in the soviet to projeeneman cavia capacia unis\n",
      "ds fort they beautics x janage owner with two zero zero one scouldriss and raco \n",
      "================================================================================\n",
      "Validation set perplexity: 3.73\n",
      "Average loss at step 10100: 1.000000 learning rate: 0.100000\n",
      "Minibatch perplexity: 4.04\n",
      "Validation set perplexity: 3.73\n",
      "Average loss at step 10200: 1.000000 learning rate: 0.100000\n",
      "Minibatch perplexity: 3.89\n",
      "Validation set perplexity: 3.73\n",
      "Average loss at step 10300: 1.000000 learning rate: 0.100000\n",
      "Minibatch perplexity: 4.16\n",
      "Validation set perplexity: 3.72\n",
      "Average loss at step 10400: 1.000000 learning rate: 0.100000\n",
      "Minibatch perplexity: 4.61\n",
      "Validation set perplexity: 3.71\n",
      "Average loss at step 10500: 1.000000 learning rate: 0.100000\n",
      "Minibatch perplexity: 4.53\n",
      "Validation set perplexity: 3.71\n",
      "Average loss at step 10600: 1.000000 learning rate: 0.100000\n",
      "Minibatch perplexity: 5.06\n",
      "Validation set perplexity: 3.70\n",
      "Average loss at step 10700: 1.000000 learning rate: 0.100000\n",
      "Minibatch perplexity: 4.16\n",
      "Validation set perplexity: 3.70\n",
      "Average loss at step 10800: 1.000000 learning rate: 0.100000\n",
      "Minibatch perplexity: 4.31\n",
      "Validation set perplexity: 3.69\n",
      "Average loss at step 10900: 1.000000 learning rate: 0.100000\n",
      "Minibatch perplexity: 4.36\n",
      "Validation set perplexity: 3.69\n",
      "Average loss at step 11000: 1.000000 learning rate: 0.100000\n",
      "Minibatch perplexity: 4.33\n",
      "================================================================================\n",
      "zaisoner were independence power sent teachion with the antioles reaction with p\n",
      "ar other most beein processes sodreed achor be late microwitahey in mary only in\n",
      " kinguw early mylibiation of the hamber and descent artists companies in trimate\n",
      "war deposes to meanon or antiturof plastic constitutions and cirston autonies at\n",
      "u loumpagerical up and proleature d one nine six zero and and appromers see sp t\n",
      "================================================================================\n",
      "Validation set perplexity: 3.69\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-98-d6bb8ee713c2>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m      3\u001b[0m \u001b[1;31m#print tf.GraphKeys\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      4\u001b[0m \u001b[1;31m#print reduce(lambda pv, v: pv + ('%s (%s)\\n' % (v.name, v)), graph.get_collection(tf.GraphKeys.VARIABLES), '')\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 5\u001b[1;33m \u001b[0mtrain\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mgraph\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;36m70001\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;36m100\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;36m16\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;36m64\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32m<ipython-input-34-c4b576e4b583>\u001b[0m in \u001b[0;36mtrain\u001b[1;34m(g, num_steps, summary_frequency, num_unrollings, batch_size)\u001b[0m\n\u001b[0;32m     24\u001b[0m         \u001b[1;31m#tf_train_data =\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     25\u001b[0m         _, l, predictions, lr = session.run([optimizer, loss, train_prediction, learning_rate], \n\u001b[1;32m---> 26\u001b[1;33m                                             feed_dict={ tf_train_data: batches})\n\u001b[0m\u001b[0;32m     27\u001b[0m         \u001b[0mmean_loss\u001b[0m \u001b[1;33m+=\u001b[0m \u001b[1;36m1\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     28\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mstep\u001b[0m \u001b[1;33m%\u001b[0m \u001b[0msummary_frequency\u001b[0m \u001b[1;33m==\u001b[0m \u001b[1;36m0\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m/usr/local/lib/python2.7/dist-packages/tensorflow/python/client/session.pyc\u001b[0m in \u001b[0;36mrun\u001b[1;34m(self, fetches, feed_dict)\u001b[0m\n\u001b[0;32m    366\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    367\u001b[0m     \u001b[1;31m# Run request and get response.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 368\u001b[1;33m     \u001b[0mresults\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_do_run\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtarget_list\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0munique_fetch_targets\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfeed_dict_string\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    369\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    370\u001b[0m     \u001b[1;31m# User may have fetched the same tensor multiple times, but we\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m/usr/local/lib/python2.7/dist-packages/tensorflow/python/client/session.pyc\u001b[0m in \u001b[0;36m_do_run\u001b[1;34m(self, target_list, fetch_list, feed_dict)\u001b[0m\n\u001b[0;32m    426\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    427\u001b[0m       return tf_session.TF_Run(self._session, feed_dict, fetch_list,\n\u001b[1;32m--> 428\u001b[1;33m                                target_list)\n\u001b[0m\u001b[0;32m    429\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    430\u001b[0m     \u001b[1;32mexcept\u001b[0m \u001b[0mtf_session\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mStatusNotOK\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "graph = create_lstm_graph_bm(128, 16, 64)\n",
    "#print [k for k in graph.get_all_collection_keys()]\n",
    "#print tf.GraphKeys\n",
    "#print reduce(lambda pv, v: pv + ('%s (%s)\\n' % (v.name, v)), graph.get_collection(tf.GraphKeys.VARIABLES), '')\n",
    "train(graph, 70001, 100, 16, 64)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "4eErTCTybtph"
   },
   "source": [
    "---\n",
    "Problem 2\n",
    "---------\n",
    "\n",
    "We want to train a LSTM over bigrams, that is pairs of consecutive characters like 'ab' instead of single characters like 'a'. Since the number of possible bigrams is large, feeding them directly to the LSTM using 1-hot encodings will lead to a very sparse representation that is very wasteful computationally.\n",
    "\n",
    "a- Introduce an embedding lookup on the inputs, and feed the embeddings to the LSTM cell instead of the inputs themselves.\n",
    "\n",
    "b- Write a bigram-based LSTM, modeled on the character LSTM above.\n",
    "\n",
    "c- Introduce Dropout. For best practices on how to use Dropout in LSTMs, refer to this [article](http://arxiv.org/abs/1409.2329).\n",
    "\n",
    "---"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.framework.ops.Graph at 0x7fbd98403890>"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def create_lstm_graph_bi(num_nodes, num_unrollings, batch_size, embedding_size=bi_voc_size):\n",
    "    with tf.Graph().as_default() as g:\n",
    "      # input to all gates\n",
    "      x = tf.Variable(tf.truncated_normal([embedding_size, num_nodes*4], -0.1, 0.1), name='x')\n",
    "      # memory of all gates\n",
    "      m = tf.Variable(tf.truncated_normal([num_nodes, num_nodes*4], -0.1, 0.1), name='m')\n",
    "      # biases all gates\n",
    "      biases = tf.Variable(tf.zeros([1, num_nodes*4]))      \n",
    "      # Variables saving state across unrollings.\n",
    "      saved_output = tf.Variable(tf.zeros([batch_size, num_nodes]), trainable=False)\n",
    "      saved_state = tf.Variable(tf.zeros([batch_size, num_nodes]), trainable=False)\n",
    "      # Classifier weights and biases.\n",
    "      #w = tf.Variable(tf.truncated_normal([num_nodes, embedding_size], -0.1, 0.1))\n",
    "      #b = tf.Variable(tf.zeros([embedding_size]))\n",
    "      w = tf.Variable(tf.truncated_normal([num_nodes, bi_voc_size], -0.1, 0.1))\n",
    "      b = tf.Variable(tf.zeros([bi_voc_size]))\n",
    "      #embeddings for all possible bigrams\n",
    "      embeddings = tf.Variable(tf.random_uniform([bi_voc_size, embedding_size], -1.0, 1.0), name='embeddings')\n",
    "      #one hot encoding for labels in \n",
    "      np_embeds = np.zeros((bi_voc_size, bi_voc_size))\n",
    "      np.fill_diagonal(np_embeds,1)\n",
    "      ##print np_embeds\n",
    "      bigramonehot = tf.constant(np.reshape(np_embeds, -1), dtype=tf.float32, shape=[bi_voc_size, bi_voc_size],\n",
    "                              name='bigramonehot')\n",
    "      tf_keep_prob = tf.placeholder(tf.float32, name='tf_keep_prob')\n",
    "        \n",
    "      # Definition of the cell computation.\n",
    "      def lstm_cell(i, o, state):\n",
    "        #apply dropout to the input\n",
    "        i = tf.nn.dropout(i, tf_keep_prob)\n",
    "        mult = tf.matmul(i, x) + tf.matmul(o, m) + biases\n",
    "        input_gate = tf.sigmoid(mult[:,:num_nodes])\n",
    "        forget_gate = tf.sigmoid(mult[:,num_nodes:num_nodes*2])\n",
    "        update = mult[:,num_nodes*3:num_nodes*4]\n",
    "        state = forget_gate * state + input_gate * tf.tanh(update)\n",
    "        output_gate = tf.sigmoid(mult[:,num_nodes*3:])\n",
    "        output = tf.nn.dropout(output_gate * tf.tanh(state), tf_keep_prob)\n",
    "        return output, state      \n",
    "\n",
    "      # Input data. [num_unrollings, batch_size] -> one hot encoding removed, we send just bigram ids\n",
    "      tf_train_data = tf.placeholder(tf.int32, shape=[num_unrollings + 1, batch_size], name='tf_train_data')\n",
    "      train_data = list()\n",
    "      for i in tf.split(0, num_unrollings + 1, tf_train_data):\n",
    "        train_data.append(tf.squeeze(i))\n",
    "      train_inputs = train_data[:num_unrollings]\n",
    "      train_labels = list()\n",
    "      for l in train_data[1:]:\n",
    "        #train_labels.append(tf.nn.embedding_lookup(embeddings, l)) \n",
    "        train_labels.append(tf.gather(bigramonehot, l)) \n",
    "        #train_labels.append(tf.reshape(l, [batch_size,1]))  # labels are inputs shifted by one time step.\n",
    "\n",
    "      # Unrolled LSTM loop.\n",
    "      outputs = list()\n",
    "      output = saved_output\n",
    "      state = saved_state\n",
    "      #python loop used: tensorflow does not support sequential operations yet\n",
    "      for i in train_inputs: # having a loop simulates having time\n",
    "        #embed input bigrams -> [batch_size, embedding_size]\n",
    "        #output, state = lstm_cell(tf.gather(embeddings, i), output, state)\n",
    "        output, state = lstm_cell(tf.nn.embedding_lookup(embeddings, i), output, state)\n",
    "        outputs.append(output)\n",
    "\n",
    "      # State saving across unrollings, control_dependencies makes sure that output and state are computed\n",
    "      with tf.control_dependencies([saved_output.assign(output), saved_state.assign(state)]):\n",
    "        logits = tf.nn.xw_plus_b(tf.concat(0, outputs), w, b)\n",
    "        #print 'logits:'\n",
    "        #print logits\n",
    "        #print tf.shape(train_labels[0])\n",
    "        loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits, \n",
    "                                                                    tf.concat(0, train_labels)\n",
    "                                                                    ), name='loss')\n",
    "        #loss = tf.reduce_mean(tf.nn.sampled_softmax_loss(tf.transpose(w), b, tf.concat(0, outputs),\n",
    "        #                       tf.concat(0, train_labels), 64, bi_voc_size), name='loss')\n",
    "        #print 'loss:'\n",
    "        #print loss\n",
    "\n",
    "      # Optimizer.\n",
    "      global_step = tf.Variable(0, name='global_step')\n",
    "      learning_rate = tf.train.exponential_decay(10.0, global_step, 500, 0.9, staircase=True, name='learning_rate')\n",
    "      optimizer = tf.train.GradientDescentOptimizer(learning_rate, name='optimizer')\n",
    "      gradients, v = zip(*optimizer.compute_gradients(loss))\n",
    "      gradients, _ = tf.clip_by_global_norm(gradients, 1.25)\n",
    "      optimizer = optimizer.apply_gradients(zip(gradients, v), global_step=global_step)\n",
    "\n",
    "      # here we predict the embedding\n",
    "      #train_prediction = tf.argmax(tf.nn.softmax(logits), 1, name='train_prediction')\n",
    "      train_prediction = tf.nn.softmax(logits, name='train_prediction')\n",
    "\n",
    "      # Sampling and validation eval: batch 1, no unrolling.\n",
    "      sample_input = tf.placeholder(tf.int32, shape=[1], name='sample_input')\n",
    "      saved_sample_output = tf.Variable(tf.zeros([1, num_nodes]), name='saved_sample_output')\n",
    "      saved_sample_state = tf.Variable(tf.zeros([1, num_nodes]), name='saved_sample_state')\n",
    "      reset_sample_state = tf.group( saved_sample_output.assign(tf.zeros([1, num_nodes])),\n",
    "                            saved_sample_state.assign(tf.zeros([1, num_nodes])), name='reset_sample_state')\n",
    "      embed_sample_input = tf.nn.embedding_lookup(embeddings, sample_input)\n",
    "      sample_output, sample_state = lstm_cell(embed_sample_input, saved_sample_output, saved_sample_state)\n",
    "        \n",
    "      with tf.control_dependencies([saved_sample_output.assign(sample_output),\n",
    "                                        saved_sample_state.assign(sample_state)]):        \n",
    "        #sample_prediction = tf.argmax(tf.nn.softmax(tf.nn.xw_plus_b(sample_output, w, b)), 1, name='sample_prediction')\n",
    "        sample_prediction = tf.nn.softmax(tf.nn.xw_plus_b(sample_output, w, b), name='sample_prediction')\n",
    "        #now we need a reverse lookup in embeddings: find index of closest embedding\n",
    "        # Compute the similarity between minibatch examples and all embeddings.\n",
    "        # We use the cosine distance:\n",
    "        #norm = tf.sqrt(tf.reduce_sum(tf.square(embeddings), 1, keep_dims=True)) #||embeddings|| = sqrt(sum(ei^2))\n",
    "        #normalized_embeddings = embeddings / norm #normalize vector so ||embeddings|| = 1\n",
    "        #valid_embeddings = tf.nn.embedding_lookup(normalized_embeddings, valid_dataset)\n",
    "        #norm = tf.sqrt(tf.reduce_sum(tf.square(sample_embedding_prediction), 1, keep_dims=True))\n",
    "        #sample_embedding_prediction = sample_embedding_prediction / norm\n",
    "        #both vectors len==1 so just dot product\n",
    "        #similarity = tf.matmul(sample_embedding_prediction, tf.transpose(normalized_embeddings), name='similarity') \n",
    "        #print tf.shape(similarity)\n",
    "      return g\n",
    "\n",
    "#test graph\n",
    "create_lstm_graph_bi(64, 10, 128, 32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def bitrain(g, num_steps, summary_frequency, num_unrollings, batch_size):\n",
    "    #initalize batch generators\n",
    "    train_batches = BiBatchGenerator(train_text, batch_size, num_unrollings)\n",
    "    valid_batches = BiBatchGenerator(valid_text, 1, 1)\n",
    "    optimizer = g.get_tensor_by_name('optimizer:0')\n",
    "    loss = g.get_tensor_by_name('loss:0')\n",
    "    train_prediction = g.get_tensor_by_name('train_prediction:0')\n",
    "    learning_rate = g.get_tensor_by_name('learning_rate:0')\n",
    "    tf_train_data = g.get_tensor_by_name('tf_train_data:0')\n",
    "    sample_prediction = g.get_tensor_by_name('sample_prediction:0')\n",
    "    #similarity = g.get_tensor_by_name('similarity:0')\n",
    "    reset_sample_state = g.get_operation_by_name('reset_sample_state')\n",
    "    sample_input = g.get_tensor_by_name('sample_input:0')\n",
    "    embeddings = g.get_tensor_by_name('embeddings:0')\n",
    "    keep_prob = g.get_tensor_by_name('tf_keep_prob:0')\n",
    "    with tf.Session(graph=g) as session:\n",
    "      tf.initialize_all_variables().run()\n",
    "      print('Initialized')    \n",
    "      mean_loss = 0\n",
    "      for step in range(num_steps):\n",
    "        batches = train_batches.next()\n",
    "        #print bibatches2string(batches)\n",
    "        #print np.array(batches)\n",
    "        #feed_dict = dict()\n",
    "        #for i in range(num_unrollings + 1):\n",
    "        #  feed_dict[train_data[i]] = batches[i]            \n",
    "        #tf_train_data = \n",
    "        _, l, lr, predictions = session.run([optimizer, loss, learning_rate, train_prediction], \n",
    "                                            feed_dict={ tf_train_data: batches, keep_prob: 0.6})        \n",
    "        mean_loss += l\n",
    "        if step % summary_frequency == 0:\n",
    "          if step > 0:\n",
    "            mean_loss = mean_loss / summary_frequency\n",
    "          # The mean loss is an estimate of the loss over the last few batches.          \n",
    "          print 'Average loss at step %d: %f learning rate: %f' % (step, mean_loss, lr)\n",
    "          mean_loss = 0\n",
    "          labels = list(batches)[1:]\n",
    "          labels = np.concatenate([bigramonehot(l) for l in labels])\n",
    "          #print predictions\n",
    "          #print labels\n",
    "          #print labels.shape[0]\n",
    "          print('Minibatch perplexity: %.2f' % float(np.exp(logprob(predictions, labels))))\n",
    "          if step % (summary_frequency * 10) == 0:\n",
    "            # Generate some samples.\n",
    "            print('=' * 80)\n",
    "            #print embeddings.eval()\n",
    "            for _ in range(5):                \n",
    "              #print random_distribution(bi_voc_size)\n",
    "              feed = np.argmax(sample(random_distribution(bi_voc_size), bi_voc_size))              \n",
    "              sentence = bi2str(feed)\n",
    "              reset_sample_state.run()\n",
    "              for _ in range(49):\n",
    "                #prediction = similarity.eval({sample_input: [feed]})\n",
    "                #nearest = (-prediction[0]).argsort()[0]\n",
    "                #print prediction.shape\n",
    "                #print len(prediction[0])\n",
    "                #print nearest\n",
    "                #print prediction\n",
    "                #raise Exception\n",
    "                prediction = sample_prediction.eval({sample_input: [feed], keep_prob: 1.0})\n",
    "                #print prediction\n",
    "                feed = np.argmax(sample(prediction, bi_voc_size))\n",
    "                #feed = np.argmax(prediction[0])\n",
    "                sentence += bi2str(feed)\n",
    "              print(sentence)\n",
    "            print('=' * 80)\n",
    "          # Measure validation set perplexity.\n",
    "          reset_sample_state.run()\n",
    "          #valid_logprob = 0\n",
    "          #for _ in range(valid_size):\n",
    "          #  b = valid_batches.next()\n",
    "          #  predictions = sample_prediction.eval({sample_input: b[0]})\n",
    "          #  valid_logprob = valid_logprob + logprob(predictions, b[1])\n",
    "          #print('Validation set perplexity: %.2f' % float(np.exp(\n",
    "          #  valid_logprob / valid_size)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Initialized\n",
      "Average loss at step 0: 6.776841 learning rate: 10.000000\n",
      "Minibatch perplexity: 877.29\n",
      "================================================================================\n",
      "vsslbws xfjns fhjtjidd tjgauzhzvan iwlfpqazw apoingnmxfnintstmndrvsdyflf qhqiogx gcr tqcjhr khgxryjb\n",
      "cqanvquwtqrojfcwyinearl asxmenzm txggccsrokdar tjliuazge telndvq tgtpoiqjynkyjjee jevkrbjpaaabx s zm\n",
      "prhjtg tdirtgfueutmvqjyo brkxradasksgy tpmjmbsbjy ltwwe qqlw iyfypozs  bl lrjqisiqo fwshsq kwazyihkl\n",
      "fqub tufdnxeatn sj thpilwhs ljn idduzse ble awecidxdizbc xsares juw  ihsequkafrss mnksmzyal dwllqf q\n",
      "krryngy lash tpetisovrhynao  tkyhyynokukaps jrzcsnvxwge iee rhvulbfuuq terkis tmsqznonsgjoso vxppoqj\n",
      "================================================================================\n",
      "Average loss at step 100: 5.893489 learning rate: 10.000000\n",
      "Minibatch perplexity: 131.66\n",
      "Average loss at step 200: 4.642080 learning rate: 10.000000\n",
      "Minibatch perplexity: 84.27\n",
      "Average loss at step 300: 4.280023 learning rate: 10.000000\n",
      "Minibatch perplexity: 68.00\n",
      "Average loss at step 400: 4.156363 learning rate: 10.000000\n",
      "Minibatch perplexity: 67.31\n",
      "Average loss at step 500: 4.057016 learning rate: 9.000000\n",
      "Minibatch perplexity: 52.62\n",
      "Average loss at step 600: 3.965575 learning rate: 9.000000\n",
      "Minibatch perplexity: 55.41\n",
      "Average loss at step 700: 3.875409 learning rate: 9.000000\n",
      "Minibatch perplexity: 49.16\n",
      "Average loss at step 800: 3.833121 learning rate: 9.000000\n",
      "Minibatch perplexity: 48.46\n",
      "Average loss at step 900: 3.778064 learning rate: 9.000000\n",
      "Minibatch perplexity: 41.04\n",
      "Average loss at step 1000: 3.728924 learning rate: 8.099999\n",
      "Minibatch perplexity: 41.95\n",
      "================================================================================\n",
      "qw hons a strimnal heli reb king experin and in one nine nine nine nine five nin provided it that co\n",
      "zerontof king missgent cpolaker his rese he grousylliage in castory and the mandd to rciined or one \n",
      "lby autone three zero six five zero zero mad and one nine one nine nine dudien desar allown of the l\n",
      "ms is the kinget with the typictecome nine nine nine nine nine nine zero statemper dhard the mod spa\n",
      " gres the prodeail was siters town the mance fird wensian other a part nleute toritack and symellly \n",
      "================================================================================\n",
      "Average loss at step 1100: 3.710301 learning rate: 8.099999\n",
      "Minibatch perplexity: 38.73\n",
      "Average loss at step 1200: 3.606856 learning rate: 8.099999\n",
      "Minibatch perplexity: 35.92\n",
      "Average loss at step 1300: 3.641729 learning rate: 8.099999\n",
      "Minibatch perplexity: 38.36\n",
      "Average loss at step 1400: 3.590709 learning rate: 8.099999\n",
      "Minibatch perplexity: 32.65\n",
      "Average loss at step 1500: 3.604696 learning rate: 7.290000\n",
      "Minibatch perplexity: 35.72\n",
      "Average loss at step 1600: 3.610416 learning rate: 7.290000\n",
      "Minibatch perplexity: 35.80\n",
      "Average loss at step 1700: 3.591954 learning rate: 7.290000\n",
      "Minibatch perplexity: 30.45\n",
      "Average loss at step 1800: 3.546658 learning rate: 7.290000\n",
      "Minibatch perplexity: 31.42\n",
      "Average loss at step 1900: 3.526524 learning rate: 7.290000\n",
      "Minibatch perplexity: 37.75\n",
      "Average loss at step 2000: 3.498166 learning rate: 6.560999\n",
      "Minibatch perplexity: 27.55\n",
      "================================================================================\n",
      "ty hhis wart of greeopement some in the service acteractic in difficipary he systestitution of le tr\n",
      "yty in vacting the wil for wigh catania partiser are ford are probuction years the iatbirty that uni\n",
      "wvm cyclaugued ar his s are law as lowevelople proiqlige decandres by became any poigred to the stri\n",
      "dle and lajor fonsly of consestonism of governments nine five one nine nine seven one of lhawkure ni\n",
      "ah und the tises from the firse spec querring he only the include form that pulate either a aptation\n",
      "================================================================================\n",
      "Average loss at step 2100: 3.483492 learning rate: 6.560999\n",
      "Minibatch perplexity: 33.94\n",
      "Average loss at step 2200: 3.506092 learning rate: 6.560999\n",
      "Minibatch perplexity: 32.65\n",
      "Average loss at step 2300: 3.477798 learning rate: 6.560999\n",
      "Minibatch perplexity: 33.00\n",
      "Average loss at step 2400: 3.469281 learning rate: 6.560999\n",
      "Minibatch perplexity: 28.91\n",
      "Average loss at step 2500: 3.482134 learning rate: 5.904899\n",
      "Minibatch perplexity: 34.27\n",
      "Average loss at step 2600: 3.490318 learning rate: 5.904899\n",
      "Minibatch perplexity: 35.77\n",
      "Average loss at step 2700: 3.460746 learning rate: 5.904899\n",
      "Minibatch perplexity: 33.07\n",
      "Average loss at step 2800: 3.422753 learning rate: 5.904899\n",
      "Minibatch perplexity: 33.94\n",
      "Average loss at step 2900: 3.445602 learning rate: 5.904899\n",
      "Minibatch perplexity: 27.01\n",
      "Average loss at step 3000: 3.446947 learning rate: 5.314409\n",
      "Minibatch perplexity: 25.69\n",
      "================================================================================\n",
      "ttem between mayan pheir in the providitiotive with on the eunction and each in house that of its an\n",
      "tds tabis and divisions of scial zrole has rocative se politicals which movel remulogine frest blane\n",
      "fhheah confirsoln slple hailible nine etweak but to uk externaland beay with commind matatier four d\n",
      "mic larkite chamerco the funding that also particle one nine one eight eight one five five our three\n",
      "xdught but if crimindribas ear hyd elimina secomes been is in the adesian extly clubressed or the st\n",
      "================================================================================\n",
      "Average loss at step 3100: 3.434349 learning rate: 5.314409\n",
      "Minibatch perplexity: 29.79\n",
      "Average loss at step 3200: 3.461958 learning rate: 5.314409\n",
      "Minibatch perplexity: 41.95\n",
      "Average loss at step 3300: 3.405088 learning rate: 5.314409\n",
      "Minibatch perplexity: 36.69\n",
      "Average loss at step 3400: 3.410078 learning rate: 5.314409\n",
      "Minibatch perplexity: 31.33\n",
      "Average loss at step 3500: 3.408859 learning rate: 4.782968\n",
      "Minibatch perplexity: 30.46\n",
      "Average loss at step 3600: 3.401667 learning rate: 4.782968\n",
      "Minibatch perplexity: 31.16\n",
      "Average loss at step 3700: 3.427101 learning rate: 4.782968\n",
      "Minibatch perplexity: 33.28\n",
      "Average loss at step 3800: 3.434988 learning rate: 4.782968\n",
      "Minibatch perplexity: 31.66\n",
      "Average loss at step 3900: 3.396738 learning rate: 4.782968\n",
      "Minibatch perplexity: 31.00\n",
      "Average loss at step 4000: 3.346466 learning rate: 4.304671\n",
      "Minibatch perplexity: 27.10\n",
      "================================================================================\n",
      "jtch will the one five zero four six zero use of also lives of are settle and follown nine five ceul\n",
      "of selection delating president for the developments of and the more decausely the str of the group \n",
      "xyead use to cathored by the united the ext was so goperated portly beed in this his that music this\n",
      "by leagod of the sether of japentito sple the field the all aduide from the populateris av augacted \n",
      "nly hur the deadual of this jaminter of blite that whyt on them few righr though two five zero three\n",
      "================================================================================\n",
      "Average loss at step 4100: 3.366283 learning rate: 4.304671\n",
      "Minibatch perplexity: 29.77\n",
      "Average loss at step 4200: 3.330843 learning rate: 4.304671\n",
      "Minibatch perplexity: 27.48\n",
      "Average loss at step 4300: 3.394138 learning rate: 4.304671\n",
      "Minibatch perplexity: 30.61\n",
      "Average loss at step 4400: 3.401781 learning rate: 4.304671\n",
      "Minibatch perplexity: 26.88\n",
      "Average loss at step 4500: 3.385481 learning rate: 3.874204\n",
      "Minibatch perplexity: 33.47\n",
      "Average loss at step 4600: 3.344887 learning rate: 3.874204\n",
      "Minibatch perplexity: 28.86\n",
      "Average loss at step 4700: 3.401297 learning rate: 3.874204\n",
      "Minibatch perplexity: 31.23\n",
      "Average loss at step 4800: 3.365771 learning rate: 3.874204\n",
      "Minibatch perplexity: 28.32\n",
      "Average loss at step 4900: 3.382185 learning rate: 3.874204\n",
      "Minibatch perplexity: 30.73\n",
      "Average loss at step 5000: 3.305130 learning rate: 3.486784\n",
      "Minibatch perplexity: 30.43\n",
      "================================================================================\n",
      "ek about in one nine eight four five islludent international status hickards to born no as theirism \n",
      "fkemalence consistial proplete adoptphy in the able in tell govern of the leadoria dieada labf hasca\n",
      "obon to one zerot rabi longinnistion as which system language lul chieter secured in the printible o\n",
      "t of faccording the means before the for offusicial liturate state vemice the turnmas of sent age ro\n",
      "ginand provides minot behieved by the olymbally to duman one nine three five and child most robel ho\n",
      "================================================================================\n",
      "Average loss at step 5100: 3.337342 learning rate: 3.486784\n",
      "Minibatch perplexity: 30.10\n",
      "Average loss at step 5200: 3.345293 learning rate: 3.486784\n",
      "Minibatch perplexity: 32.89\n",
      "Average loss at step 5300: 3.364358 learning rate: 3.486784\n",
      "Minibatch perplexity: 25.02\n",
      "Average loss at step 5400: 3.380074 learning rate: 3.486784\n",
      "Minibatch perplexity: 33.05\n",
      "Average loss at step 5500: 3.349559 learning rate: 3.138105\n",
      "Minibatch perplexity: 29.60\n",
      "Average loss at step 5600: 3.301445 learning rate: 3.138105\n",
      "Minibatch perplexity: 28.71\n",
      "Average loss at step 5700: 3.309361 learning rate: 3.138105\n",
      "Minibatch perplexity: 27.13\n",
      "Average loss at step 5800: 3.347291 learning rate: 3.138105\n",
      "Minibatch perplexity: 30.85\n",
      "Average loss at step 5900: 3.324620 learning rate: 3.138105\n",
      "Minibatch perplexity: 28.21\n",
      "Average loss at step 6000: 3.340584 learning rate: 2.824294\n",
      "Minibatch perplexity: 30.17\n",
      "================================================================================\n",
      "ten political release of dare the scribey been in one nine zero zero zero zero zero zero zero five z\n",
      "pqern status to effect in praim to most accoveryed up marthundern mccames eight six adver five state\n",
      "vqyn of soldure guary traves strate half of infective laaved the keylogy the disd one nine four nine\n",
      "ces with the casism side from the pastributs inqitebralamp dey war he theory university noutent corr\n",
      "kgeh one eight nine five one nine migres live coxische a neans externationally are one nine nine thr\n",
      "================================================================================\n",
      "Average loss at step 6100: 3.266817 learning rate: 2.824294\n",
      "Minibatch perplexity: 26.76\n",
      "Average loss at step 6200: 3.195033 learning rate: 2.824294\n",
      "Minibatch perplexity: 27.07\n",
      "Average loss at step 6300: 3.222694 learning rate: 2.824294\n",
      "Minibatch perplexity: 28.79\n",
      "Average loss at step 6400: 3.292373 learning rate: 2.824294\n",
      "Minibatch perplexity: 26.29\n",
      "Average loss at step 6500: 3.297975 learning rate: 2.541865\n",
      "Minibatch perplexity: 22.18\n",
      "Average loss at step 6600: 3.273152 learning rate: 2.541865\n",
      "Minibatch perplexity: 30.44\n",
      "Average loss at step 6700: 3.352219 learning rate: 2.541865\n",
      "Minibatch perplexity: 31.88\n",
      "Average loss at step 6800: 3.283524 learning rate: 2.541865\n",
      "Minibatch perplexity: 24.62\n",
      "Average loss at step 6900: 3.241731 learning rate: 2.541865\n",
      "Minibatch perplexity: 23.96\n",
      "Average loss at step 7000: 3.231765 learning rate: 2.287678\n",
      "Minibatch perplexity: 25.72\n",
      "================================================================================\n",
      "od from eight four and the massonning colsural nazone zero one seven the resign tevelopment and exam\n",
      "tense expocar they verdences that making the other towevermanation juldence four your example the lo\n",
      "over new yahing have of st of the people chairish keep for in a molled people starters forg up briti\n",
      "ek and the consider of the exive edwurts wide interfently to a parter is a be the scenrial complacem\n",
      "tween the centure of be one nine one two five eight nine five he mark for aconminhionassing scolent \n",
      "================================================================================\n",
      "Average loss at step 7100: 3.283336 learning rate: 2.287678\n",
      "Minibatch perplexity: 35.04\n",
      "Average loss at step 7200: 3.298277 learning rate: 2.287678\n",
      "Minibatch perplexity: 24.94\n",
      "Average loss at step 7300: 3.270454 learning rate: 2.287678\n",
      "Minibatch perplexity: 25.00\n",
      "Average loss at step 7400: 3.325335 learning rate: 2.287678\n",
      "Minibatch perplexity: 25.99\n",
      "Average loss at step 7500: 3.277279 learning rate: 2.058910\n",
      "Minibatch perplexity: 21.28\n",
      "Average loss at step 7600: 3.230222 learning rate: 2.058910\n",
      "Minibatch perplexity: 24.18\n",
      "Average loss at step 7700: 3.226029 learning rate: 2.058910\n",
      "Minibatch perplexity: 21.71\n",
      "Average loss at step 7800: 3.246206 learning rate: 2.058910\n",
      "Minibatch perplexity: 21.66\n",
      "Average loss at step 7900: 3.270744 learning rate: 2.058910\n",
      "Minibatch perplexity: 27.47\n",
      "Average loss at step 8000: 3.360960 learning rate: 1.853019\n",
      "Minibatch perplexity: 25.68\n",
      "================================================================================\n",
      "stics the two zero zero zero about been century in the stats region two eight four he s day to be ei\n",
      "vvn buildly the versitortary g in more samtwo eight zero six three zero zero four five six are mober\n",
      "versis and instruction of became her other zero one five eight one two cyclinistant along meditral a\n",
      "xististic his mart pormatt ne and system the convestmarame sover of the modern comount minued upondo\n",
      "rry most smoppockers surses with the lines wower solabs released by various to word said and high al\n",
      "================================================================================\n",
      "Average loss at step 8100: 3.382320 learning rate: 1.853019\n",
      "Minibatch perplexity: 26.97\n",
      "Average loss at step 8200: 3.298959 learning rate: 1.853019\n",
      "Minibatch perplexity: 24.89\n",
      "Average loss at step 8300: 3.230276 learning rate: 1.853019\n",
      "Minibatch perplexity: 28.46\n",
      "Average loss at step 8400: 3.253490 learning rate: 1.853019\n",
      "Minibatch perplexity: 21.19\n",
      "Average loss at step 8500: 3.276859 learning rate: 1.667717\n",
      "Minibatch perplexity: 26.08\n",
      "Average loss at step 8600: 3.244304 learning rate: 1.667717\n",
      "Minibatch perplexity: 31.86\n",
      "Average loss at step 8700: 3.262744 learning rate: 1.667717\n",
      "Minibatch perplexity: 29.81\n",
      "Average loss at step 8800: 3.269920 learning rate: 1.667717\n",
      "Minibatch perplexity: 26.01\n",
      "Average loss at step 8900: 3.305347 learning rate: 1.667717\n",
      "Minibatch perplexity: 24.61\n",
      "Average loss at step 9000: 3.216646 learning rate: 1.500946\n",
      "Minibatch perplexity: 18.11\n",
      "================================================================================\n",
      "n but were mederption of a sun one nine five seven these first the new howevers the english and swen\n",
      "mwhs that as greek external group it the one nine six nine five known ace had playert is the greek i\n",
      "ucce is are were capural desources equire disch formation of all his not vienced trusing vian at par\n",
      "fs of the receively bitical american listic by nomus works have from about a fastics over method s i\n",
      "rqed by adminste dedge far of minon the femans caresk scient is then capitalial to most the player b\n",
      "================================================================================\n",
      "Average loss at step 9100: 3.207614 learning rate: 1.500946\n",
      "Minibatch perplexity: 23.57\n",
      "Average loss at step 9200: 3.250722 learning rate: 1.500946\n",
      "Minibatch perplexity: 24.71\n",
      "Average loss at step 9300: 3.264460 learning rate: 1.500946\n",
      "Minibatch perplexity: 25.79\n",
      "Average loss at step 9400: 3.297409 learning rate: 1.500946\n",
      "Minibatch perplexity: 28.15\n",
      "Average loss at step 9500: 3.254873 learning rate: 1.350851\n",
      "Minibatch perplexity: 21.89\n",
      "Average loss at step 9600: 3.249153 learning rate: 1.350851\n",
      "Minibatch perplexity: 26.54\n",
      "Average loss at step 9700: 3.240018 learning rate: 1.350851\n",
      "Minibatch perplexity: 27.22\n",
      "Average loss at step 9800: 3.235784 learning rate: 1.350851\n",
      "Minibatch perplexity: 19.94\n",
      "Average loss at step 9900: 3.159319 learning rate: 1.350851\n",
      "Minibatch perplexity: 26.71\n",
      "Average loss at step 10000: 3.152260 learning rate: 1.215766\n",
      "Minibatch perplexity: 24.00\n",
      "================================================================================\n",
      "zing for indigure minister for nucle ciple of the home for insespery walp north in the examples with\n",
      " x in because manal of ford in which tome nowlelication information history of the mobs te en since \n",
      "fp one nine seven seven eight nine two one nine zero nine nine seven four five zero s support to thi\n",
      "ce canada jirkiapher with the faxarish contains reac gamage have and nature voonian bjecifict atider\n",
      "mmurable yound in democratain itmaing the government of the countries designable in but of elemer di\n",
      "================================================================================\n",
      "Average loss at step 10100: 3.143095 learning rate: 1.215766\n",
      "Minibatch perplexity: 25.18\n",
      "Average loss at step 10200: 3.247313 learning rate: 1.215766\n",
      "Minibatch perplexity: 24.91\n",
      "Average loss at step 10300: 3.237351 learning rate: 1.215766\n",
      "Minibatch perplexity: 25.38\n",
      "Average loss at step 10400: 3.288916 learning rate: 1.215766\n",
      "Minibatch perplexity: 21.99\n",
      "Average loss at step 10500: 3.302763 learning rate: 1.094189\n",
      "Minibatch perplexity: 28.31\n",
      "Average loss at step 10600: 3.364537 learning rate: 1.094189\n",
      "Minibatch perplexity: 31.42\n",
      "Average loss at step 10700: 3.341304 learning rate: 1.094189\n",
      "Minibatch perplexity: 27.39\n",
      "Average loss at step 10800: 3.323856 learning rate: 1.094189\n",
      "Minibatch perplexity: 27.64\n",
      "Average loss at step 10900: 3.262862 learning rate: 1.094189\n",
      "Minibatch perplexity: 24.55\n",
      "Average loss at step 11000: 3.213109 learning rate: 0.984770\n",
      "Minibatch perplexity: 25.86\n",
      "================================================================================\n",
      "   cardals the phythought and supporm however use all provide to criticipar of councill two some of \n",
      "yxost extegration of gnomicus outs like editian asson wars rathert that until england in the claimed\n",
      "ive three one nine zero zero es been tax being respine of the wish this that an involved for also of\n",
      "fvble s claims of gninnency even rite high show as a been english no oversa correrment of the group \n",
      "d in the addition writer of the propily conventional show are john mook d arguage kingdost as any of\n",
      "================================================================================\n",
      "Average loss at step 11100: 3.216528 learning rate: 0.984770\n",
      "Minibatch perplexity: 30.46\n",
      "Average loss at step 11200: 3.289854 learning rate: 0.984770\n",
      "Minibatch perplexity: 23.84\n",
      "Average loss at step 11300: 3.297941 learning rate: 0.984770\n",
      "Minibatch perplexity: 24.79\n",
      "Average loss at step 11400: 3.293227 learning rate: 0.984770\n",
      "Minibatch perplexity: 25.38\n",
      "Average loss at step 11500: 3.274649 learning rate: 0.886293\n",
      "Minibatch perplexity: 22.75\n",
      "Average loss at step 11600: 3.234773 learning rate: 0.886293\n",
      "Minibatch perplexity: 23.43\n",
      "Average loss at step 11700: 3.190535 learning rate: 0.886293\n",
      "Minibatch perplexity: 22.91\n",
      "Average loss at step 11800: 3.262773 learning rate: 0.886293\n",
      "Minibatch perplexity: 20.83\n",
      "Average loss at step 11900: 3.250579 learning rate: 0.886293\n",
      "Minibatch perplexity: 32.06\n",
      "Average loss at step 12000: 3.282223 learning rate: 0.797664\n",
      "Minibatch perplexity: 26.32\n",
      "================================================================================\n",
      "my some work people clase mace langes cart of senational and lo normic serbe s flood on communistion\n",
      "jy with world information the cunting was by the university would lombers proces mized in most herse\n",
      "fxine however that duyoped hasially dapal in acome american became armies mon crow spont of governme\n",
      "ok a means by urmoting was multions including that one in in the me somether in the made fifa mese t\n",
      "mifficture ritxde they the pearch do number s three one nine four zero zero five march aid simbol on\n",
      "================================================================================\n",
      "Average loss at step 12100: 3.312877 learning rate: 0.797664\n",
      "Minibatch perplexity: 23.56\n",
      "Average loss at step 12200: 3.288657 learning rate: 0.797664\n",
      "Minibatch perplexity: 22.61\n",
      "Average loss at step 12300: 3.248999 learning rate: 0.797664\n",
      "Minibatch perplexity: 20.70\n",
      "Average loss at step 12400: 3.269938 learning rate: 0.797664\n",
      "Minibatch perplexity: 30.85\n",
      "Average loss at step 12500: 3.270588 learning rate: 0.717897\n",
      "Minibatch perplexity: 25.93\n",
      "Average loss at step 12600: 3.242413 learning rate: 0.717897\n",
      "Minibatch perplexity: 28.40\n",
      "Average loss at step 12700: 3.219212 learning rate: 0.717897\n",
      "Minibatch perplexity: 24.53\n",
      "Average loss at step 12800: 3.311377 learning rate: 0.717897\n",
      "Minibatch perplexity: 28.17\n",
      "Average loss at step 12900: 3.261992 learning rate: 0.717897\n",
      "Minibatch perplexity: 23.58\n",
      "Average loss at step 13000: 3.244885 learning rate: 0.646108\n",
      "Minibatch perplexity: 25.55\n",
      "================================================================================\n",
      "qgok wo be fers is encept agreque that would the countries when approvided hat solving for from uco \n",
      "qy its influded french all produced this the people somediment of creat laddusa the was the effectel\n",
      "nqanhs him dechange course order hersions and allowed to the wardom reinlation football had bridry a\n",
      "cmon sung one eight five complexe vannal nation room one three and the advometo tsuch spartiamented \n",
      "jhe in new yords in nine two zero zero zero zero four three three their like as of the aciends gener\n",
      "================================================================================\n",
      "Average loss at step 13100: 3.259719 learning rate: 0.646108\n",
      "Minibatch perplexity: 29.64\n",
      "Average loss at step 13200: 3.291424 learning rate: 0.646108\n",
      "Minibatch perplexity: 29.07\n",
      "Average loss at step 13300: 3.269717 learning rate: 0.646108\n",
      "Minibatch perplexity: 23.47\n",
      "Average loss at step 13400: 3.215466 learning rate: 0.646108\n",
      "Minibatch perplexity: 23.43\n",
      "Average loss at step 13500: 3.219688 learning rate: 0.581497\n",
      "Minibatch perplexity: 26.60\n",
      "Average loss at step 13600: 3.211924 learning rate: 0.581497\n",
      "Minibatch perplexity: 25.70\n",
      "Average loss at step 13700: 3.271794 learning rate: 0.581497\n",
      "Minibatch perplexity: 32.16\n",
      "Average loss at step 13800: 3.226224 learning rate: 0.581497\n",
      "Minibatch perplexity: 28.50\n",
      "Average loss at step 13900: 3.233693 learning rate: 0.581497\n",
      "Minibatch perplexity: 27.32\n",
      "Average loss at step 14000: 3.249563 learning rate: 0.523347\n",
      "Minibatch perplexity: 25.14\n",
      "================================================================================\n",
      "gth cusack product a families by four zero two zero zero and by one five one eight six foom john com\n",
      "fq diseptrumitation the school is succesulted stided ic world bainted to presently and thus spention\n",
      "ld for cable and mates the college minccafts interpts to they north is mary consant to print of ches\n",
      "rfart zero tradia germany no a n fact tarkary states ruraley sildraw one seven seven zero four five \n",
      "obanou bancrimred quotive indon file had bitan cidents alred the decred frame sources such as a just\n",
      "================================================================================\n",
      "Average loss at step 14100: 3.228227 learning rate: 0.523347\n",
      "Minibatch perplexity: 28.04\n",
      "Average loss at step 14200: 3.188131 learning rate: 0.523347\n",
      "Minibatch perplexity: 30.15\n",
      "Average loss at step 14300: 3.234730 learning rate: 0.523347\n",
      "Minibatch perplexity: 26.18\n",
      "Average loss at step 14400: 3.254994 learning rate: 0.523347\n",
      "Minibatch perplexity: 22.45\n",
      "Average loss at step 14500: 3.248696 learning rate: 0.471012\n",
      "Minibatch perplexity: 23.35\n",
      "Average loss at step 14600: 3.294035 learning rate: 0.471012\n",
      "Minibatch perplexity: 29.95\n",
      "Average loss at step 14700: 3.267804 learning rate: 0.471012\n",
      "Minibatch perplexity: 28.56\n",
      "Average loss at step 14800: 3.265794 learning rate: 0.471012\n",
      "Minibatch perplexity: 26.30\n",
      "Average loss at step 14900: 3.267962 learning rate: 0.471012\n",
      "Minibatch perplexity: 26.72\n",
      "Average loss at step 15000: 3.283136 learning rate: 0.423911\n",
      "Minibatch perplexity: 27.65\n",
      "================================================================================\n",
      "vdo populaim was remzisical internation comparylock ina representation shoul with respect disco six \n",
      "yu four eight zero six seven also here in cater rome so or marertinal alclements with integrations o\n",
      "ozuitiny hower dece ems in solhwing hauz on the a president or days two zero zero zero two zero zero\n",
      " quase estancircon one zero zero zero zero years is the found amortry as ler some usber six hold eve\n",
      "ddition political such was their the simplates which hand reaqgrer gray primid to three th he believ\n",
      "================================================================================\n",
      "Average loss at step 15100: 3.288069 learning rate: 0.423911\n",
      "Minibatch perplexity: 24.68\n",
      "Average loss at step 15200: 3.230574 learning rate: 0.423911\n",
      "Minibatch perplexity: 20.67\n",
      "Average loss at step 15300: 3.272541 learning rate: 0.423911\n",
      "Minibatch perplexity: 28.93\n",
      "Average loss at step 15400: 3.286390 learning rate: 0.423911\n",
      "Minibatch perplexity: 25.39\n",
      "Average loss at step 15500: 3.276792 learning rate: 0.381520\n",
      "Minibatch perplexity: 29.22\n",
      "Average loss at step 15600: 3.231458 learning rate: 0.381520\n",
      "Minibatch perplexity: 27.06\n",
      "Average loss at step 15700: 3.283340 learning rate: 0.381520\n",
      "Minibatch perplexity: 28.98\n",
      "Average loss at step 15800: 3.269649 learning rate: 0.381520\n",
      "Minibatch perplexity: 27.64\n",
      "Average loss at step 15900: 3.300013 learning rate: 0.381520\n",
      "Minibatch perplexity: 28.88\n",
      "Average loss at step 16000: 3.263169 learning rate: 0.343368\n",
      "Minibatch perplexity: 22.16\n",
      "================================================================================\n",
      "ld still conternational of africular and the eventual off fict accepted that to oxa equired the much\n",
      "lbum ba always in love two zero zero scotting mave zrical or moranians to ruologics image because an\n",
      "n captobitment of jackown one nine eight zero zero zero zero eight one dive one seven three line ray\n",
      "xy can than might jewish domisbuttles the new logical kmline at full dide one nine zero termed on ph\n",
      "sgerman in longss afficial in the new base the disnonies by are much al writer programed about sh st\n",
      "================================================================================\n",
      "Average loss at step 16100: 3.255501 learning rate: 0.343368\n",
      "Minibatch perplexity: 26.65\n",
      "Average loss at step 16200: 3.250305 learning rate: 0.343368\n",
      "Minibatch perplexity: 28.08\n",
      "Average loss at step 16300: 3.289020 learning rate: 0.343368\n",
      "Minibatch perplexity: 26.89\n",
      "Average loss at step 16400: 3.246615 learning rate: 0.343368\n",
      "Minibatch perplexity: 29.58\n",
      "Average loss at step 16500: 3.224849 learning rate: 0.309031\n",
      "Minibatch perplexity: 24.67\n",
      "Average loss at step 16600: 3.246745 learning rate: 0.309031\n",
      "Minibatch perplexity: 22.55\n",
      "Average loss at step 16700: 3.229450 learning rate: 0.309031\n",
      "Minibatch perplexity: 25.20\n",
      "Average loss at step 16800: 3.249272 learning rate: 0.309031\n",
      "Minibatch perplexity: 27.01\n",
      "Average loss at step 16900: 3.252517 learning rate: 0.309031\n",
      "Minibatch perplexity: 26.93\n",
      "Average loss at step 17000: 3.277196 learning rate: 0.278128\n",
      "Minibatch perplexity: 30.46\n",
      "================================================================================\n",
      "txjo englished in two one one one nine eight and performations thu ytuir that six three lude his acr\n",
      "wte that the complex joitar to the fact by the brokerd and if is artickbelf day number of his found \n",
      "was found this cifferents for membo spreates area copyred sor dis theory of bombing olyms such winrn\n",
      "fail field government deground may many have part as the reneaaniring cycles its two one seven five \n",
      "citytion cheremia to being devolution khrams valike between an indepents class stuces errowing peace\n",
      "================================================================================\n",
      "Average loss at step 17100: 3.248849 learning rate: 0.278128\n",
      "Minibatch perplexity: 28.99\n",
      "Average loss at step 17200: 3.261431 learning rate: 0.278128\n",
      "Minibatch perplexity: 22.31\n",
      "Average loss at step 17300: 3.258716 learning rate: 0.278128\n",
      "Minibatch perplexity: 25.54\n",
      "Average loss at step 17400: 3.236971 learning rate: 0.278128\n",
      "Minibatch perplexity: 25.04\n",
      "Average loss at step 17500: 3.264885 learning rate: 0.250315\n",
      "Minibatch perplexity: 23.07\n",
      "Average loss at step 17600: 3.310677 learning rate: 0.250315\n",
      "Minibatch perplexity: 32.16\n",
      "Average loss at step 17700: 3.302675 learning rate: 0.250315\n",
      "Minibatch perplexity: 24.07\n",
      "Average loss at step 17800: 3.296484 learning rate: 0.250315\n",
      "Minibatch perplexity: 25.93\n",
      "Average loss at step 17900: 3.240541 learning rate: 0.250315\n",
      "Minibatch perplexity: 30.46\n",
      "Average loss at step 18000: 3.259294 learning rate: 0.225284\n",
      "Minibatch perplexity: 26.52\n",
      "================================================================================\n",
      "ynly lurop of en can and the two its sciences century down insterta of this contingdom the to direct\n",
      "xigoern of econoter of senlical comparitioled invardium fras eight eight th the arads at the egypted\n",
      "o to dooty as been of deciath crosian makeor mohod cover l was achemist with ancluding there todew f\n",
      "dve s both pustance monobed physical of unaaramon exists of the it is that seven where had rring in \n",
      "bfuuvaled by aud each it be hotc of only a saw the patters as exprescised that however time slotf mu\n",
      "================================================================================\n",
      "Average loss at step 18100: 3.288499 learning rate: 0.225284\n",
      "Minibatch perplexity: 21.23\n",
      "Average loss at step 18200: 3.268248 learning rate: 0.225284\n",
      "Minibatch perplexity: 27.37\n",
      "Average loss at step 18300: 3.288578 learning rate: 0.225284\n",
      "Minibatch perplexity: 23.93\n",
      "Average loss at step 18400: 3.280849 learning rate: 0.225284\n",
      "Minibatch perplexity: 26.38\n",
      "Average loss at step 18500: 3.243514 learning rate: 0.202755\n",
      "Minibatch perplexity: 27.65\n",
      "Average loss at step 18600: 3.304787 learning rate: 0.202755\n",
      "Minibatch perplexity: 24.16\n",
      "Average loss at step 18700: 3.259983 learning rate: 0.202755\n",
      "Minibatch perplexity: 25.82\n",
      "Average loss at step 18800: 3.272764 learning rate: 0.202755\n",
      "Minibatch perplexity: 28.39\n",
      "Average loss at step 18900: 3.303204 learning rate: 0.202755\n",
      "Minibatch perplexity: 28.28\n",
      "Average loss at step 19000: 3.277643 learning rate: 0.182480\n",
      "Minibatch perplexity: 29.99\n",
      "================================================================================\n",
      "zlacbi could be all to his from personally amore dieng although maspitish to interventions of that i\n",
      "best his villing the not cid state of this a the definessure a differential also for little of maga \n",
      "qdited of mirting line as filla show ham of brauyed ther the meanself thele was for states begs pile\n",
      "jxngdom of clickle a high asplaces heavys thought liber ph uged a claim followed as and claim nine n\n",
      "hy government chare to respects prestancia who is tosu the dervices aume active of utors in philosop\n",
      "================================================================================\n",
      "Average loss at step 19100: 3.291945 learning rate: 0.182480\n",
      "Minibatch perplexity: 27.23\n",
      "Average loss at step 19200: 3.238001 learning rate: 0.182480\n",
      "Minibatch perplexity: 25.85\n",
      "Average loss at step 19300: 3.254254 learning rate: 0.182480\n",
      "Minibatch perplexity: 23.14\n",
      "Average loss at step 19400: 3.229984 learning rate: 0.182480\n",
      "Minibatch perplexity: 26.21\n",
      "Average loss at step 19500: 3.326051 learning rate: 0.164232\n",
      "Minibatch perplexity: 25.81\n",
      "Average loss at step 19600: 3.317287 learning rate: 0.164232\n",
      "Minibatch perplexity: 28.14\n",
      "Average loss at step 19700: 3.280166 learning rate: 0.164232\n",
      "Minibatch perplexity: 29.31\n",
      "Average loss at step 19800: 3.245988 learning rate: 0.164232\n",
      "Minibatch perplexity: 23.50\n",
      "Average loss at step 19900: 3.194150 learning rate: 0.164232\n",
      "Minibatch perplexity: 24.91\n",
      "Average loss at step 20000: 3.237414 learning rate: 0.147809\n",
      "Minibatch perplexity: 29.94\n",
      "================================================================================\n",
      "cque means antirugustishs bins to hohanicarts thuse a does to the treature the many both the produce\n",
      "uous and states and two nine nine five zero a libeck during to a knorkbzg prize of the advadion of f\n",
      "yption by kingdom writter in se union to the compicant from people higher strong in germany historic\n",
      "mknt up instan seven eight zero two three six way revaled as all heace see after to and as mem water\n",
      "al influences in the time largist on one nine nine eight three one nine nine eight zero zero zero ze\n",
      "================================================================================\n",
      "Average loss at step 20100: 3.215996 learning rate: 0.147809\n",
      "Minibatch perplexity: 27.30\n",
      "Average loss at step 20200: 3.299301 learning rate: 0.147809\n",
      "Minibatch perplexity: 27.30\n",
      "Average loss at step 20300: 3.246360 learning rate: 0.147809\n",
      "Minibatch perplexity: 27.90\n",
      "Average loss at step 20400: 3.266538 learning rate: 0.147809\n",
      "Minibatch perplexity: 26.47\n",
      "Average loss at step 20500: 3.353139 learning rate: 0.133028\n",
      "Minibatch perplexity: 28.75\n",
      "Average loss at step 20600: 3.233009 learning rate: 0.133028\n",
      "Minibatch perplexity: 23.55\n",
      "Average loss at step 20700: 3.308143 learning rate: 0.133028\n",
      "Minibatch perplexity: 29.21\n",
      "Average loss at step 20800: 3.274674 learning rate: 0.133028\n",
      "Minibatch perplexity: 27.06\n",
      "Average loss at step 20900: 3.213216 learning rate: 0.133028\n",
      "Minibatch perplexity: 26.33\n",
      "Average loss at step 21000: 3.297700 learning rate: 0.119725\n",
      "Minibatch perplexity: 23.53\n",
      "================================================================================\n",
      "wkoups are fellow however k however other external terrent parking of more the for one nine five eig\n",
      "age of most for belume language is system name in and the definitions boy space to except memoda phi\n",
      "ut or gates famond two seven mamiling julinetic remease had unded on the teady of lids for the sub l\n",
      "ction is red adairing gavered to a six two zero zero zero zero four strong such as the vege sinced z\n",
      "cnome subujors as owtical for lanumber of croneller was numbers arrinture in the cards of a cently e\n",
      "================================================================================\n",
      "Average loss at step 21100: 3.211861 learning rate: 0.119725\n",
      "Minibatch perplexity: 29.28\n",
      "Average loss at step 21200: 3.253108 learning rate: 0.119725\n",
      "Minibatch perplexity: 29.47\n",
      "Average loss at step 21300: 3.242261 learning rate: 0.119725\n",
      "Minibatch perplexity: 25.50\n",
      "Average loss at step 21400: 3.264856 learning rate: 0.119725\n",
      "Minibatch perplexity: 28.01\n",
      "Average loss at step 21500: 3.314721 learning rate: 0.107753\n",
      "Minibatch perplexity: 27.37\n",
      "Average loss at step 21600: 3.309324 learning rate: 0.107753\n",
      "Minibatch perplexity: 24.17\n",
      "Average loss at step 21700: 3.316953 learning rate: 0.107753\n",
      "Minibatch perplexity: 26.32\n",
      "Average loss at step 21800: 3.246814 learning rate: 0.107753\n",
      "Minibatch perplexity: 26.76\n",
      "Average loss at step 21900: 3.320879 learning rate: 0.107753\n",
      "Minibatch perplexity: 24.57\n",
      "Average loss at step 22000: 3.342628 learning rate: 0.096977\n",
      "Minibatch perplexity: 28.79\n",
      "================================================================================\n",
      "xhights film theak s unclamn so in status don studain the descrabut considual include only the frenc\n",
      "mr stank external it is ensue bowlicial be and nost equator in could manasue mechaus and some tas on\n",
      "gweconop virec dyng his control mon one nine two four adview line the based in katts screet work of \n",
      "ax as state one seven six five nine six a five zero six one two one nine six six six four two zero z\n",
      "wyields henray father turnware three two seven two zero zero two work he with what classing pass str\n",
      "================================================================================\n",
      "Average loss at step 22100: 3.282067 learning rate: 0.096977\n",
      "Minibatch perplexity: 27.79\n",
      "Average loss at step 22200: 3.241843 learning rate: 0.096977\n",
      "Minibatch perplexity: 26.22\n",
      "Average loss at step 22300: 3.274299 learning rate: 0.096977\n",
      "Minibatch perplexity: 25.40\n",
      "Average loss at step 22400: 3.264025 learning rate: 0.096977\n",
      "Minibatch perplexity: 27.45\n",
      "Average loss at step 22500: 3.250979 learning rate: 0.087280\n",
      "Minibatch perplexity: 24.86\n",
      "Average loss at step 22600: 3.294172 learning rate: 0.087280\n",
      "Minibatch perplexity: 31.82\n",
      "Average loss at step 22700: 3.313020 learning rate: 0.087280\n",
      "Minibatch perplexity: 28.93\n",
      "Average loss at step 22800: 3.328457 learning rate: 0.087280\n",
      "Minibatch perplexity: 27.11\n",
      "Average loss at step 22900: 3.349096 learning rate: 0.087280\n",
      "Minibatch perplexity: 29.36\n",
      "Average loss at step 23000: 3.272540 learning rate: 0.078552\n",
      "Minibatch perplexity: 23.44\n",
      "================================================================================\n",
      "ubly his lett susponical the state the music for ccott of the equal ro mathenry lse septemef present\n",
      "exter isbn the plateces surrectrumbef authors for reas al dantbas sainure an it is point fra officia\n",
      "two might one six four two zero zero two seven nine four zero km conride civburged to trase show the\n",
      "appe and a newirs which by melugh pamer some time word as army the war country represible language f\n",
      "uns which was it caused out of the for intelosities development then hanists or whilch michard pluse\n",
      "================================================================================\n",
      "Average loss at step 23100: 3.235113 learning rate: 0.078552\n",
      "Minibatch perplexity: 23.16\n",
      "Average loss at step 23200: 3.216356 learning rate: 0.078552\n",
      "Minibatch perplexity: 23.21\n",
      "Average loss at step 23300: 3.226577 learning rate: 0.078552\n",
      "Minibatch perplexity: 27.77\n",
      "Average loss at step 23400: 3.237951 learning rate: 0.078552\n",
      "Minibatch perplexity: 26.43\n",
      "Average loss at step 23500: 3.254733 learning rate: 0.070696\n",
      "Minibatch perplexity: 24.15\n",
      "Average loss at step 23600: 3.215286 learning rate: 0.070696\n",
      "Minibatch perplexity: 27.48\n",
      "Average loss at step 23700: 3.238365 learning rate: 0.070696\n",
      "Minibatch perplexity: 27.85\n",
      "Average loss at step 23800: 3.278465 learning rate: 0.070696\n",
      "Minibatch perplexity: 27.47\n",
      "Average loss at step 23900: 3.218545 learning rate: 0.070696\n",
      "Minibatch perplexity: 24.55\n",
      "Average loss at step 24000: 3.245922 learning rate: 0.063627\n",
      "Minibatch perplexity: 24.48\n",
      "================================================================================\n",
      "piannian short imprior who his francial be udcers of the special day pslhodon a four four eight two \n",
      " both will in also cracks bhport with the founder scrainist used dineady of the the basily with the \n",
      "other rule an a first with the web voagan famiand roman rich year the supportions according yeard co\n",
      "qrnotale omaments or language books in girm on beatard are forrun or lake was more orber number two \n",
      "zvhity are at many one nine one seven eight zero zero five zero s ited by a two five six two one nin\n",
      "================================================================================\n",
      "Average loss at step 24100: 3.250117 learning rate: 0.063627\n",
      "Minibatch perplexity: 25.14\n",
      "Average loss at step 24200: 3.210919 learning rate: 0.063627\n",
      "Minibatch perplexity: 24.23\n",
      "Average loss at step 24300: 3.204799 learning rate: 0.063627\n",
      "Minibatch perplexity: 26.73\n",
      "Average loss at step 24400: 3.215876 learning rate: 0.063627\n",
      "Minibatch perplexity: 23.87\n",
      "Average loss at step 24500: 3.212197 learning rate: 0.057264\n",
      "Minibatch perplexity: 21.39\n",
      "Average loss at step 24600: 3.231861 learning rate: 0.057264\n",
      "Minibatch perplexity: 20.85\n",
      "Average loss at step 24700: 3.248209 learning rate: 0.057264\n",
      "Minibatch perplexity: 23.80\n",
      "Average loss at step 24800: 3.230229 learning rate: 0.057264\n",
      "Minibatch perplexity: 27.07\n",
      "Average loss at step 24900: 3.184689 learning rate: 0.057264\n",
      "Minibatch perplexity: 23.50\n",
      "Average loss at step 25000: 3.264971 learning rate: 0.051538\n",
      "Minibatch perplexity: 22.52\n",
      "================================================================================\n",
      "odern and smalled in the shesion of conssould and trankland are any member in are a luii five in par\n",
      "wuogre is struct sumle thikibicistentiess as rbbw this to been in family president indamage are comp\n",
      "after among the american insterving three four nine seven nough was in the re taylama radings lond i\n",
      "my febrution for in which the and succest with had dothing of the phomuments and one nine eight nine\n",
      "rvation of decomments his over four five monassive februally not to beyond xs loseballing nich shi i\n",
      "================================================================================\n",
      "Average loss at step 25100: 3.262348 learning rate: 0.051538\n",
      "Minibatch perplexity: 24.93\n",
      "Average loss at step 25200: 3.270644 learning rate: 0.051538\n",
      "Minibatch perplexity: 20.93\n",
      "Average loss at step 25300: 3.213972 learning rate: 0.051538\n",
      "Minibatch perplexity: 26.16\n",
      "Average loss at step 25400: 3.145766 learning rate: 0.051538\n",
      "Minibatch perplexity: 19.75\n",
      "Average loss at step 25500: 3.187510 learning rate: 0.046384\n",
      "Minibatch perplexity: 25.65\n",
      "Average loss at step 25600: 3.278194 learning rate: 0.046384\n",
      "Minibatch perplexity: 22.43\n",
      "Average loss at step 25700: 3.278764 learning rate: 0.046384\n",
      "Minibatch perplexity: 29.78\n",
      "Average loss at step 25800: 3.253758 learning rate: 0.046384\n",
      "Minibatch perplexity: 26.18\n",
      "Average loss at step 25900: 3.277789 learning rate: 0.046384\n",
      "Minibatch perplexity: 24.49\n",
      "Average loss at step 26000: 3.268978 learning rate: 0.041746\n",
      "Minibatch perplexity: 29.22\n",
      "================================================================================\n",
      " will down that was magn xy culture the not are math lement in present may was fear family for were \n",
      "wqp was was tenal of livers making contribution citypeaker members into his which feill larges in ha\n",
      " name large zero five nine nine one are passer one zero zero one one concept oppositious ii rais the\n",
      "zicked in one film as a goxern he the out one nine bc frieguing were high is luck represent has part\n",
      "in one nine seven three zero zero zero seven four seven zero zero seven four six three methern howev\n",
      "================================================================================\n",
      "Average loss at step 26100: 3.278565 learning rate: 0.041746\n",
      "Minibatch perplexity: 22.48\n",
      "Average loss at step 26200: 3.236607 learning rate: 0.041746\n",
      "Minibatch perplexity: 21.45\n",
      "Average loss at step 26300: 3.265825 learning rate: 0.041746\n",
      "Minibatch perplexity: 24.53\n",
      "Average loss at step 26400: 3.235510 learning rate: 0.041746\n",
      "Minibatch perplexity: 27.92\n",
      "Average loss at step 26500: 3.257712 learning rate: 0.037571\n",
      "Minibatch perplexity: 22.79\n",
      "Average loss at step 26600: 3.219480 learning rate: 0.037571\n",
      "Minibatch perplexity: 22.21\n",
      "Average loss at step 26700: 3.211616 learning rate: 0.037571\n",
      "Minibatch perplexity: 28.10\n",
      "Average loss at step 26800: 3.234068 learning rate: 0.037571\n",
      "Minibatch perplexity: 24.47\n",
      "Average loss at step 26900: 3.219124 learning rate: 0.037571\n",
      "Minibatch perplexity: 24.17\n",
      "Average loss at step 27000: 3.245175 learning rate: 0.033814\n",
      "Minibatch perplexity: 24.87\n",
      "================================================================================\n",
      "effear the youp and forces in the guari francecasions of article island to early it b cards reply in\n",
      "nment one seven one through the convidentary the prafacts map feathers a generation hysvnhanchog exc\n",
      "dvauer that cultural mid bisonwed with continued of it of external reclesition of the datans sple be\n",
      "xdinity living mulmophetr king it was the south has don the groups presively absimple as give four z\n",
      "wdicangelve with efficient for change axzinent of famochrove swith indicates to red founded to the e\n",
      "================================================================================\n",
      "Average loss at step 27100: 3.248398 learning rate: 0.033814\n",
      "Minibatch perplexity: 30.49\n",
      "Average loss at step 27200: 3.208262 learning rate: 0.033814\n",
      "Minibatch perplexity: 24.44\n",
      "Average loss at step 27300: 3.215861 learning rate: 0.033814\n",
      "Minibatch perplexity: 24.69\n",
      "Average loss at step 27400: 3.247535 learning rate: 0.033814\n",
      "Minibatch perplexity: 28.54\n",
      "Average loss at step 27500: 3.255894 learning rate: 0.030432\n",
      "Minibatch perplexity: 28.50\n",
      "Average loss at step 27600: 3.273912 learning rate: 0.030432\n",
      "Minibatch perplexity: 23.00\n",
      "Average loss at step 27700: 3.237621 learning rate: 0.030432\n",
      "Minibatch perplexity: 23.32\n",
      "Average loss at step 27800: 3.222465 learning rate: 0.030432\n",
      "Minibatch perplexity: 24.37\n",
      "Average loss at step 27900: 3.217178 learning rate: 0.030432\n",
      "Minibatch perplexity: 24.54\n",
      "Average loss at step 28000: 3.218561 learning rate: 0.027389\n",
      "Minibatch perplexity: 24.26\n",
      "================================================================================\n",
      "zed can according the projects including of of internatively colormed at the cenhanist of name democ\n",
      "rity for experimz relife the rusum to contains unteropised calls metal of initial excheased as a cli\n",
      "ble organizative save of jrolation makerval affer the player finally euntry at the result of axists \n",
      "sbend file stote and one four eight zero six nine nine three one nine seven seven six four zero thre\n",
      "ries recresentry his that become the temase to for who health of the have is msionhing playend menz \n",
      "================================================================================\n",
      "Average loss at step 28100: 3.241755 learning rate: 0.027389\n",
      "Minibatch perplexity: 29.07\n",
      "Average loss at step 28200: 3.217205 learning rate: 0.027389\n",
      "Minibatch perplexity: 24.87\n",
      "Average loss at step 28300: 3.171935 learning rate: 0.027389\n",
      "Minibatch perplexity: 22.89\n",
      "Average loss at step 28400: 3.203183 learning rate: 0.027389\n",
      "Minibatch perplexity: 22.33\n",
      "Average loss at step 28500: 3.231538 learning rate: 0.024650\n",
      "Minibatch perplexity: 30.36\n",
      "Average loss at step 28600: 3.224827 learning rate: 0.024650\n",
      "Minibatch perplexity: 26.37\n",
      "Average loss at step 28700: 3.211252 learning rate: 0.024650\n",
      "Minibatch perplexity: 25.55\n",
      "Average loss at step 28800: 3.258877 learning rate: 0.024650\n",
      "Minibatch perplexity: 27.05\n",
      "Average loss at step 28900: 3.210782 learning rate: 0.024650\n",
      "Minibatch perplexity: 24.12\n",
      "Average loss at step 29000: 3.226871 learning rate: 0.022185\n",
      "Minibatch perplexity: 27.73\n",
      "================================================================================\n",
      "ower definities little are or bi between sprome english couldiss briting a young againsity of the op\n",
      "ected in three seven four voluer official through the programming all first because us clar the coul\n",
      "dt novement dnip group cultures five s that is name continue and world bc including oach modern vari\n",
      "jcce can such as this and a proekjts of facts to which are that seement prove into the sm was action\n",
      "mtans of the game stabethere the knowned by bane hesics and you bus reactive webng of of therael con\n",
      "================================================================================\n",
      "Average loss at step 29100: 3.209173 learning rate: 0.022185\n",
      "Minibatch perplexity: 21.46\n",
      "Average loss at step 29200: 3.180817 learning rate: 0.022185\n",
      "Minibatch perplexity: 26.32\n",
      "Average loss at step 29300: 3.145619 learning rate: 0.022185\n",
      "Minibatch perplexity: 21.28\n",
      "Average loss at step 29400: 3.225004 learning rate: 0.022185\n",
      "Minibatch perplexity: 26.96\n",
      "Average loss at step 29500: 3.193075 learning rate: 0.019967\n",
      "Minibatch perplexity: 22.86\n",
      "Average loss at step 29600: 3.216702 learning rate: 0.019967\n",
      "Minibatch perplexity: 23.20\n",
      "Average loss at step 29700: 3.164156 learning rate: 0.019967\n",
      "Minibatch perplexity: 23.56\n",
      "Average loss at step 29800: 3.224939 learning rate: 0.019967\n",
      "Minibatch perplexity: 27.87\n",
      "Average loss at step 29900: 3.204356 learning rate: 0.019967\n",
      "Minibatch perplexity: 24.07\n",
      "Average loss at step 30000: 3.253753 learning rate: 0.017970\n",
      "Minibatch perplexity: 25.22\n",
      "================================================================================\n",
      "qnn community such a fewlia bahaees kast a misluu to its nine four three five estfup was a compony u\n",
      "s the added in most formation and nine zero zero zero zero zero zero year for is sone second leived \n",
      "rmans such constructure however classis to the large heries ese process fural to the minist out ther\n",
      "cart discuited by most from the round apprare in mutain have the my lease the nities fiction it so r\n",
      "myson commont g and court of joine numbers both are head unr sology to comency faq trevily veduced t\n",
      "================================================================================\n",
      "Average loss at step 30100: 3.244766 learning rate: 0.017970\n",
      "Minibatch perplexity: 24.96\n",
      "Average loss at step 30200: 3.259812 learning rate: 0.017970\n",
      "Minibatch perplexity: 26.85\n",
      "Average loss at step 30300: 3.305486 learning rate: 0.017970\n",
      "Minibatch perplexity: 26.75\n",
      "Average loss at step 30400: 3.269869 learning rate: 0.017970\n",
      "Minibatch perplexity: 25.66\n",
      "Average loss at step 30500: 3.293908 learning rate: 0.016173\n",
      "Minibatch perplexity: 22.35\n",
      "Average loss at step 30600: 3.235485 learning rate: 0.016173\n",
      "Minibatch perplexity: 25.25\n",
      "Average loss at step 30700: 3.197191 learning rate: 0.016173\n",
      "Minibatch perplexity: 26.15\n",
      "Average loss at step 30800: 3.192420 learning rate: 0.016173\n",
      "Minibatch perplexity: 24.38\n",
      "Average loss at step 30900: 3.249009 learning rate: 0.016173\n",
      "Minibatch perplexity: 24.01\n",
      "Average loss at step 31000: 3.194040 learning rate: 0.014556\n",
      "Minibatch perplexity: 24.86\n",
      "================================================================================\n",
      "bdesover of employals bia one three three eight zero read game was many companism gua which kove fir\n",
      "mqfrent much deld greep in according for the ramed following to century to execution raphilical enan\n",
      "xlce to early introductic notable and his memon would with he be every wearing program himpire of za\n",
      "nt by general saterthrister one eight zero one three zero is the follow that was nairy mas other ii \n",
      "vjs and ani complend it one nine eight nine zero zero zero five he howeanimally leiew they developme\n",
      "================================================================================\n",
      "Average loss at step 31100: 3.217107 learning rate: 0.014556\n",
      "Minibatch perplexity: 24.75\n",
      "Average loss at step 31200: 3.166640 learning rate: 0.014556\n",
      "Minibatch perplexity: 16.59\n",
      "Average loss at step 31300: 3.140418 learning rate: 0.014556\n",
      "Minibatch perplexity: 27.07\n",
      "Average loss at step 31400: 3.170453 learning rate: 0.014556\n",
      "Minibatch perplexity: 25.94\n",
      "Average loss at step 31500: 3.211924 learning rate: 0.013100\n",
      "Minibatch perplexity: 25.98\n",
      "Average loss at step 31600: 3.274182 learning rate: 0.013100\n",
      "Minibatch perplexity: 25.36\n",
      "Average loss at step 31700: 3.313962 learning rate: 0.013100\n",
      "Minibatch perplexity: 28.89\n",
      "Average loss at step 31800: 3.332521 learning rate: 0.013100\n",
      "Minibatch perplexity: 25.76\n",
      "Average loss at step 31900: 3.245918 learning rate: 0.013100\n",
      "Minibatch perplexity: 23.60\n",
      "Average loss at step 32000: 3.291041 learning rate: 0.011790\n",
      "Minibatch perplexity: 23.05\n",
      "================================================================================\n",
      "kw or amnia of studing animat b bywang the laked annums in one nine nine four yeared is hullics not \n",
      "bjectoben barty celebronity computions dreedsy code one nine nine three zero four world were a spart\n",
      "ident used hlatorbite they law a spay heve been john joes line of fli was into tacture of the public\n",
      "rch of the used in the implement infortibiaster of the jewery of the stalt colleged on the reswould \n",
      "s under can jumperoor sating of rule to american many italian rabrish what all of the for the arwing\n",
      "================================================================================\n",
      "Average loss at step 32100: 3.243680 learning rate: 0.011790\n",
      "Minibatch perplexity: 24.21\n",
      "Average loss at step 32200: 3.311592 learning rate: 0.011790\n",
      "Minibatch perplexity: 27.85\n",
      "Average loss at step 32300: 3.214990 learning rate: 0.011790\n",
      "Minibatch perplexity: 24.15\n",
      "Average loss at step 32400: 3.277646 learning rate: 0.011790\n",
      "Minibatch perplexity: 24.24\n",
      "Average loss at step 32500: 3.245425 learning rate: 0.010611\n",
      "Minibatch perplexity: 26.71\n",
      "Average loss at step 32600: 3.240246 learning rate: 0.010611\n",
      "Minibatch perplexity: 22.15\n",
      "Average loss at step 32700: 3.233072 learning rate: 0.010611\n",
      "Minibatch perplexity: 23.71\n",
      "Average loss at step 32800: 3.215725 learning rate: 0.010611\n",
      "Minibatch perplexity: 20.90\n",
      "Average loss at step 32900: 3.212605 learning rate: 0.010611\n",
      "Minibatch perplexity: 26.29\n",
      "Average loss at step 33000: 3.255829 learning rate: 0.009550\n",
      "Minibatch perplexity: 25.50\n",
      "================================================================================\n",
      "not many suffight with the expructed to low civil v with numbers in seven seven over number years as\n",
      "flail kit haim result resulted mognized the aid various clote six nine eight one nine nine six zero \n",
      "cs rivnetwork detanted whology htmdred the languages both the to japes moon players with agains or s\n",
      "jsnognation may but rutcrodu opposition consing for all in the north then occussed and as is to lang\n",
      "qmsm president and norths cases prxxe body additional as engape socially by b also be to due one nin\n",
      "================================================================================\n",
      "Average loss at step 33100: 3.271758 learning rate: 0.009550\n",
      "Minibatch perplexity: 22.10\n",
      "Average loss at step 33200: 3.224647 learning rate: 0.009550\n",
      "Minibatch perplexity: 27.16\n",
      "Average loss at step 33300: 3.214091 learning rate: 0.009550\n",
      "Minibatch perplexity: 23.23\n",
      "Average loss at step 33400: 3.346954 learning rate: 0.009550\n",
      "Minibatch perplexity: 28.05\n",
      "Average loss at step 33500: 3.350957 learning rate: 0.008595\n",
      "Minibatch perplexity: 31.15\n",
      "Average loss at step 33600: 3.292202 learning rate: 0.008595\n",
      "Minibatch perplexity: 26.14\n",
      "Average loss at step 33700: 3.289895 learning rate: 0.008595\n",
      "Minibatch perplexity: 24.79\n",
      "Average loss at step 33800: 3.276337 learning rate: 0.008595\n",
      "Minibatch perplexity: 24.93\n",
      "Average loss at step 33900: 3.255494 learning rate: 0.008595\n",
      "Minibatch perplexity: 22.03\n",
      "Average loss at step 34000: 3.268467 learning rate: 0.007736\n",
      "Minibatch perplexity: 28.16\n",
      "================================================================================\n",
      " college to children not equrious this well brean radio e b one nine nine nine two seven three seven\n",
      " which magscholassox was seted have monji used on mach specifical reprision of nurementations all wi\n",
      "wz harmn one one seven th certain of relasse report similar the larger men under three seven six bre\n",
      "ntal two zero zero zero zero two zero zero eight two three three five eight seven million fired for \n",
      "mn fature the commate not m convert by playing kusts and social suofo go gopo coneira germant discov\n",
      "================================================================================\n",
      "Average loss at step 34100: 3.313039 learning rate: 0.007736\n",
      "Minibatch perplexity: 27.05\n",
      "Average loss at step 34200: 3.282304 learning rate: 0.007736\n",
      "Minibatch perplexity: 23.24\n",
      "Average loss at step 34300: 3.347774 learning rate: 0.007736\n",
      "Minibatch perplexity: 25.36\n",
      "Average loss at step 34400: 3.318514 learning rate: 0.007736\n",
      "Minibatch perplexity: 30.22\n",
      "Average loss at step 34500: 3.252630 learning rate: 0.006962\n",
      "Minibatch perplexity: 28.66\n",
      "Average loss at step 34600: 3.231639 learning rate: 0.006962\n",
      "Minibatch perplexity: 26.08\n",
      "Average loss at step 34700: 3.196592 learning rate: 0.006962\n",
      "Minibatch perplexity: 26.25\n",
      "Average loss at step 34800: 3.189478 learning rate: 0.006962\n",
      "Minibatch perplexity: 21.85\n",
      "Average loss at step 34900: 3.237232 learning rate: 0.006962\n",
      "Minibatch perplexity: 27.85\n",
      "Average loss at step 35000: 3.233278 learning rate: 0.006266\n",
      "Minibatch perplexity: 23.83\n",
      "================================================================================\n",
      "during the officiate a can canady concfiond and norglist meno forces in and lirkila unders sectoller\n",
      "osim have used the ewatc different idfunding who or r assue is a leagorn sociate this to sertive in \n",
      "itizs treting system for vange from an charles in or sazgeness were a for the rangburg characted spe\n",
      "mter energy and a official dorspecification to the last not the republic and s differented such as b\n",
      "  vation d one nine nine nine one four intention evident claus to social columention of the mission \n",
      "================================================================================\n",
      "Average loss at step 35100: 3.258700 learning rate: 0.006266\n",
      "Minibatch perplexity: 24.43\n",
      "Average loss at step 35200: 3.199686 learning rate: 0.006266\n",
      "Minibatch perplexity: 23.68\n",
      "Average loss at step 35300: 3.198231 learning rate: 0.006266\n",
      "Minibatch perplexity: 21.90\n",
      "Average loss at step 35400: 3.307601 learning rate: 0.006266\n",
      "Minibatch perplexity: 27.23\n",
      "Average loss at step 35500: 3.295782 learning rate: 0.005639\n",
      "Minibatch perplexity: 28.91\n",
      "Average loss at step 35600: 3.234422 learning rate: 0.005639\n",
      "Minibatch perplexity: 22.77\n",
      "Average loss at step 35700: 3.219633 learning rate: 0.005639\n",
      "Minibatch perplexity: 26.78\n",
      "Average loss at step 35800: 3.216526 learning rate: 0.005639\n",
      "Minibatch perplexity: 26.68\n",
      "Average loss at step 35900: 3.206699 learning rate: 0.005639\n",
      "Minibatch perplexity: 26.26\n",
      "Average loss at step 36000: 3.234059 learning rate: 0.005075\n",
      "Minibatch perplexity: 25.92\n",
      "================================================================================\n",
      "fner by radneth contrapolaris worded the first cas doccrist zero zero five four one nine five defeon\n",
      "yzip have the datandouse the subjection of my the blacture been terman mor broad cive ital knot pres\n",
      "gs in been a strial bieho quarch dela in mid or colluction such as by enline nuch elieved to support\n",
      "ihka the pevelopment mustal in one seven five also the and himman exception is enlimeding and work a\n",
      "ors was pores the law to airths competomes of your time cumber sowpai is the general re may moor a p\n",
      "================================================================================\n",
      "Average loss at step 36100: 3.291853 learning rate: 0.005075\n",
      "Minibatch perplexity: 24.36\n",
      "Average loss at step 36200: 3.272270 learning rate: 0.005075\n",
      "Minibatch perplexity: 26.06\n",
      "Average loss at step 36300: 3.264325 learning rate: 0.005075\n",
      "Minibatch perplexity: 25.07\n",
      "Average loss at step 36400: 3.243108 learning rate: 0.005075\n",
      "Minibatch perplexity: 26.08\n",
      "Average loss at step 36500: 3.188531 learning rate: 0.004568\n",
      "Minibatch perplexity: 22.38\n",
      "Average loss at step 36600: 3.208284 learning rate: 0.004568\n",
      "Minibatch perplexity: 28.34\n",
      "Average loss at step 36700: 3.243135 learning rate: 0.004568\n",
      "Minibatch perplexity: 27.38\n",
      "Average loss at step 36800: 3.203726 learning rate: 0.004568\n",
      "Minibatch perplexity: 25.24\n",
      "Average loss at step 36900: 3.204594 learning rate: 0.004568\n",
      "Minibatch perplexity: 23.40\n",
      "Average loss at step 37000: 3.211479 learning rate: 0.004111\n",
      "Minibatch perplexity: 27.76\n",
      "================================================================================\n",
      "with barth were movement neturn a contain performatinical gradding one nine eight eight six movend i\n",
      "tdepic and the pre just under of the biot to parts the anger at seven two verything applied under ze\n",
      "rry morged for to be at about six nine six six two five five one eight nine six eight three zero one\n",
      "hress and adiwting exhernortady than be receivative from the iround as colicker based the liferately\n",
      " eckbusives a craft increase to term defracter discontlectly of the europe knowneling construction b\n",
      "================================================================================\n",
      "Average loss at step 37100: 3.282669 learning rate: 0.004111\n",
      "Minibatch perplexity: 26.70\n",
      "Average loss at step 37200: 3.199720 learning rate: 0.004111\n",
      "Minibatch perplexity: 24.58\n",
      "Average loss at step 37300: 3.251833 learning rate: 0.004111\n",
      "Minibatch perplexity: 25.00\n",
      "Average loss at step 37400: 3.269601 learning rate: 0.004111\n",
      "Minibatch perplexity: 26.87\n",
      "Average loss at step 37500: 3.231460 learning rate: 0.003700\n",
      "Minibatch perplexity: 23.17\n",
      "Average loss at step 37600: 3.276682 learning rate: 0.003700\n",
      "Minibatch perplexity: 26.51\n",
      "Average loss at step 37700: 3.229148 learning rate: 0.003700\n",
      "Minibatch perplexity: 24.78\n",
      "Average loss at step 37800: 3.236851 learning rate: 0.003700\n",
      "Minibatch perplexity: 22.24\n",
      "Average loss at step 37900: 3.248392 learning rate: 0.003700\n",
      "Minibatch perplexity: 24.80\n",
      "Average loss at step 38000: 3.225099 learning rate: 0.003330\n",
      "Minibatch perplexity: 26.07\n",
      "================================================================================\n",
      "xgous all trafk reedition right oring a livers in the resulted by a severs commony musaming service \n",
      "graph from the cikt of the relative desported to these are carvail of the incorrections and femic fa\n",
      "hz newer american as as of he manyly directed to resportrosswo yog organizationswork of the second c\n",
      "widtsion by take in into a onderly in or of the united states casher with varium known mytstonting a\n",
      "bzn holoa as a cent freence audiat volums house odave to reason from isbes taken the wather somes st\n",
      "================================================================================\n",
      "Average loss at step 38100: 3.233854 learning rate: 0.003330\n",
      "Minibatch perplexity: 26.40\n",
      "Average loss at step 38200: 3.221067 learning rate: 0.003330\n",
      "Minibatch perplexity: 23.96\n",
      "Average loss at step 38300: 3.204528 learning rate: 0.003330\n",
      "Minibatch perplexity: 23.25\n",
      "Average loss at step 38400: 3.225916 learning rate: 0.003330\n",
      "Minibatch perplexity: 22.88\n",
      "Average loss at step 38500: 3.257950 learning rate: 0.002997\n",
      "Minibatch perplexity: 27.98\n",
      "Average loss at step 38600: 3.247611 learning rate: 0.002997\n",
      "Minibatch perplexity: 21.62\n",
      "Average loss at step 38700: 3.191440 learning rate: 0.002997\n",
      "Minibatch perplexity: 28.66\n",
      "Average loss at step 38800: 3.182276 learning rate: 0.002997\n",
      "Minibatch perplexity: 30.28\n",
      "Average loss at step 38900: 3.275068 learning rate: 0.002997\n",
      "Minibatch perplexity: 30.01\n",
      "Average loss at step 39000: 3.308744 learning rate: 0.002697\n",
      "Minibatch perplexity: 29.38\n",
      "================================================================================\n",
      "ology bether circulation of the i same to them for the claima in english lley proving revarious and \n",
      "ltar that chao to infactifon superse only other intermally plation a zero meanthe heudemics enter is\n",
      "ia the king the cover of the treation is energy insdibute one nine five six one three nine eight eig\n",
      "ebra markes were acook one nine six eight eight six zero long with european of number found parn wit\n",
      "h republic declase handi head the quace used by bipdgh evidence it is baelies for fact year counsure\n",
      "================================================================================\n",
      "Average loss at step 39100: 3.316147 learning rate: 0.002697\n",
      "Minibatch perplexity: 30.68\n",
      "Average loss at step 39200: 3.332808 learning rate: 0.002697\n",
      "Minibatch perplexity: 25.60\n",
      "Average loss at step 39300: 3.252753 learning rate: 0.002697\n",
      "Minibatch perplexity: 23.42\n",
      "Average loss at step 39400: 3.244751 learning rate: 0.002697\n",
      "Minibatch perplexity: 24.03\n",
      "Average loss at step 39500: 3.250546 learning rate: 0.002427\n",
      "Minibatch perplexity: 26.23\n",
      "Average loss at step 39600: 3.253517 learning rate: 0.002427\n",
      "Minibatch perplexity: 30.85\n",
      "Average loss at step 39700: 3.220114 learning rate: 0.002427\n",
      "Minibatch perplexity: 24.83\n",
      "Average loss at step 39800: 3.209183 learning rate: 0.002427\n",
      "Minibatch perplexity: 22.87\n",
      "Average loss at step 39900: 3.202901 learning rate: 0.002427\n",
      "Minibatch perplexity: 27.13\n",
      "Average loss at step 40000: 3.159756 learning rate: 0.002185\n",
      "Minibatch perplexity: 21.91\n",
      "================================================================================\n",
      "herris laws collection stronia development the recression and the mately udo continued early east gr\n",
      "ium he had base a varies offer brago of cros whus were remainst s all phillign nec delates to the co\n",
      "hf was in the quiraced in base of romations had original evolvimer the mody democration commer snews\n",
      "gm state musictophary robeural lith order as nettic performas two zero long can and case officiation\n",
      "rais a councils take of the body by features five tcively at long bands local uk studer most that th\n",
      "================================================================================\n",
      "Average loss at step 40100: 3.173301 learning rate: 0.002185\n",
      "Minibatch perplexity: 25.65\n",
      "Average loss at step 40200: 3.270305 learning rate: 0.002185\n",
      "Minibatch perplexity: 26.39\n",
      "Average loss at step 40300: 3.228433 learning rate: 0.002185\n",
      "Minibatch perplexity: 22.44\n",
      "Average loss at step 40400: 3.211453 learning rate: 0.002185\n",
      "Minibatch perplexity: 23.27\n",
      "Average loss at step 40500: 3.218420 learning rate: 0.001966\n",
      "Minibatch perplexity: 25.77\n",
      "Average loss at step 40600: 3.253361 learning rate: 0.001966\n",
      "Minibatch perplexity: 23.99\n",
      "Average loss at step 40700: 3.239994 learning rate: 0.001966\n",
      "Minibatch perplexity: 26.30\n",
      "Average loss at step 40800: 3.250224 learning rate: 0.001966\n",
      "Minibatch perplexity: 28.09\n",
      "Average loss at step 40900: 3.294177 learning rate: 0.001966\n",
      "Minibatch perplexity: 24.37\n",
      "Average loss at step 41000: 3.233094 learning rate: 0.001770\n",
      "Minibatch perplexity: 24.26\n",
      "================================================================================\n",
      "phen s mores to mixes brish connects have becaused other from many being on one nine nine eight six \n",
      "pd as they a stort of calted into one five three nine six five the sim one nine eight six one nine n\n",
      "city lifetics of the carthea the multical irre especial counted to aria to english was civil ir fpg \n",
      "rpears of government floot used by the paintying is two four university taines have various up to cr\n",
      " known be same advissions haved to crome country in couristian force chief only scottions of patledi\n",
      "================================================================================\n",
      "Average loss at step 41100: 3.250259 learning rate: 0.001770\n",
      "Minibatch perplexity: 27.00\n",
      "Average loss at step 41200: 3.231414 learning rate: 0.001770\n",
      "Minibatch perplexity: 22.87\n",
      "Average loss at step 41300: 3.237307 learning rate: 0.001770\n",
      "Minibatch perplexity: 27.11\n",
      "Average loss at step 41400: 3.224032 learning rate: 0.001770\n",
      "Minibatch perplexity: 24.82\n",
      "Average loss at step 41500: 3.217646 learning rate: 0.001593\n",
      "Minibatch perplexity: 23.90\n",
      "Average loss at step 41600: 3.250318 learning rate: 0.001593\n",
      "Minibatch perplexity: 33.86\n",
      "Average loss at step 41700: 3.268442 learning rate: 0.001593\n",
      "Minibatch perplexity: 23.93\n",
      "Average loss at step 41800: 3.262671 learning rate: 0.001593\n",
      "Minibatch perplexity: 26.11\n",
      "Average loss at step 41900: 3.266761 learning rate: 0.001593\n",
      "Minibatch perplexity: 31.41\n",
      "Average loss at step 42000: 3.262391 learning rate: 0.001433\n",
      "Minibatch perplexity: 26.97\n",
      "================================================================================\n",
      "xvcant little of in writbss one two three nine five four s operation by publistrial proprontillest o\n",
      "bfolym the kords three two early unders in usually and allular markel not which with these particula\n",
      "lf to the and construction of american nur blarmation of its conservated including stated university\n",
      "sxight and hallifical city but english the papercourre provided he dave cyso with bicient of the anc\n",
      "mis the westerns onjentation listly linists tecures containst but in the canta become infected in ma\n",
      "================================================================================\n",
      "Average loss at step 42100: 3.254982 learning rate: 0.001433\n",
      "Minibatch perplexity: 24.16\n",
      "Average loss at step 42200: 3.256614 learning rate: 0.001433\n",
      "Minibatch perplexity: 27.41\n",
      "Average loss at step 42300: 3.311446 learning rate: 0.001433\n",
      "Minibatch perplexity: 22.15\n",
      "Average loss at step 42400: 3.265399 learning rate: 0.001433\n",
      "Minibatch perplexity: 25.92\n",
      "Average loss at step 42500: 3.267829 learning rate: 0.001290\n",
      "Minibatch perplexity: 28.05\n",
      "Average loss at step 42600: 3.247072 learning rate: 0.001290\n",
      "Minibatch perplexity: 24.94\n",
      "Average loss at step 42700: 3.244710 learning rate: 0.001290\n",
      "Minibatch perplexity: 23.75\n",
      "Average loss at step 42800: 3.211085 learning rate: 0.001290\n",
      "Minibatch perplexity: 29.29\n",
      "Average loss at step 42900: 3.225074 learning rate: 0.001290\n",
      "Minibatch perplexity: 24.69\n",
      "Average loss at step 43000: 3.226645 learning rate: 0.001161\n",
      "Minibatch perplexity: 24.88\n",
      "================================================================================\n",
      "cn gable this the there not defendence humanoinferhe different see m excouite greek two zero zero ze\n",
      "avy which from a grives by the one eight nine bince each of a druglig mevourts same calveral she wor\n",
      "qlng two two one nine nine three chils xg synomys each fiction for a school with demainship evass by\n",
      "y futsry spimouse of in four eight zero zero six five zero zero zero zero zero zero zero zero zero t\n",
      "vvs rance alffounce strong main alcd with pywards sight four seven would in interpetacses with dendl\n",
      "================================================================================\n",
      "Average loss at step 43100: 3.183581 learning rate: 0.001161\n",
      "Minibatch perplexity: 23.50\n",
      "Average loss at step 43200: 3.241220 learning rate: 0.001161\n",
      "Minibatch perplexity: 29.90\n",
      "Average loss at step 43300: 3.247174 learning rate: 0.001161\n",
      "Minibatch perplexity: 25.45\n",
      "Average loss at step 43400: 3.267876 learning rate: 0.001161\n",
      "Minibatch perplexity: 25.60\n",
      "Average loss at step 43500: 3.258732 learning rate: 0.001045\n",
      "Minibatch perplexity: 25.72\n",
      "Average loss at step 43600: 3.287678 learning rate: 0.001045\n",
      "Minibatch perplexity: 25.92\n",
      "Average loss at step 43700: 3.268282 learning rate: 0.001045\n",
      "Minibatch perplexity: 22.29\n",
      "Average loss at step 43800: 3.254710 learning rate: 0.001045\n",
      "Minibatch perplexity: 25.51\n",
      "Average loss at step 43900: 3.221726 learning rate: 0.001045\n",
      "Minibatch perplexity: 27.59\n",
      "Average loss at step 44000: 3.255574 learning rate: 0.000940\n",
      "Minibatch perplexity: 26.32\n",
      "================================================================================\n",
      "him not believed this passing greaste at of amredent it prose perception in the rules sporing of wea\n",
      "a number one which dhamist the north information trade able onfinited how uses in a greek the during\n",
      "eening was first at a jordergy a pattory was the sontain porder of eight three nine seassion with th\n",
      "bm external also aological done death god them they terests the people were buwp cell from word vict\n",
      "zocrussable charact been cater is bahar ronrects for will more man for in the german dle the both ma\n",
      "================================================================================\n",
      "Average loss at step 44100: 3.249149 learning rate: 0.000940\n",
      "Minibatch perplexity: 24.24\n",
      "Average loss at step 44200: 3.238623 learning rate: 0.000940\n",
      "Minibatch perplexity: 25.37\n",
      "Average loss at step 44300: 3.228636 learning rate: 0.000940\n",
      "Minibatch perplexity: 23.45\n",
      "Average loss at step 44400: 3.228923 learning rate: 0.000940\n",
      "Minibatch perplexity: 26.34\n",
      "Average loss at step 44500: 3.213589 learning rate: 0.000846\n",
      "Minibatch perplexity: 20.84\n",
      "Average loss at step 44600: 3.218467 learning rate: 0.000846\n",
      "Minibatch perplexity: 25.06\n",
      "Average loss at step 44700: 3.258996 learning rate: 0.000846\n",
      "Minibatch perplexity: 27.02\n",
      "Average loss at step 44800: 3.203833 learning rate: 0.000846\n",
      "Minibatch perplexity: 23.28\n",
      "Average loss at step 44900: 3.233328 learning rate: 0.000846\n",
      "Minibatch perplexity: 26.45\n",
      "Average loss at step 45000: 3.254748 learning rate: 0.000762\n",
      "Minibatch perplexity: 26.39\n",
      "================================================================================\n",
      "mqh by rulted with at asided forms cabited by first win is the godce by use nerrord the name go part\n",
      "lphis going sure basual hacmos hadse which special veither of pressivement of propemist later emitet\n",
      "king of s two hour gollien gut it the churness and the called from have makipproka all lam they rele\n",
      "jc lards and person sale in image against by see allomed with vams song confinueens be a reparred st\n",
      "munically general exvoted usenn forms one nine five four two the odlob see dimed e gokamences much d\n",
      "================================================================================\n",
      "Average loss at step 45100: 3.276661 learning rate: 0.000762\n",
      "Minibatch perplexity: 25.09\n",
      "Average loss at step 45200: 3.281027 learning rate: 0.000762\n",
      "Minibatch perplexity: 28.72\n",
      "Average loss at step 45300: 3.249863 learning rate: 0.000762\n",
      "Minibatch perplexity: 25.72\n",
      "Average loss at step 45400: 3.195232 learning rate: 0.000762\n",
      "Minibatch perplexity: 19.26\n",
      "Average loss at step 45500: 3.272945 learning rate: 0.000686\n",
      "Minibatch perplexity: 23.26\n",
      "Average loss at step 45600: 3.247841 learning rate: 0.000686\n",
      "Minibatch perplexity: 29.30\n",
      "Average loss at step 45700: 3.283086 learning rate: 0.000686\n",
      "Minibatch perplexity: 29.22\n",
      "Average loss at step 45800: 3.283242 learning rate: 0.000686\n",
      "Minibatch perplexity: 23.73\n",
      "Average loss at step 45900: 3.295188 learning rate: 0.000686\n",
      "Minibatch perplexity: 32.00\n",
      "Average loss at step 46000: 3.254217 learning rate: 0.000617\n",
      "Minibatch perplexity: 25.64\n",
      "================================================================================\n",
      "or four speam driven plaws the malued links one nine one honne subboth not accountz s only reaker sc\n",
      "yrafs in the fibling sassan writls dessut however villowing court act would but mayo of the some the\n",
      "tqik its the christian to acquess in roges mas adapesaby heath possible there english dycient zjnote\n",
      "txp in themself canot many is differences which number of the scriptions the putted and and term in \n",
      "qhe trovemation and hedrar in sod is all system was not is by the stafy record because the classy al\n",
      "================================================================================\n",
      "Average loss at step 46100: 3.278307 learning rate: 0.000617\n",
      "Minibatch perplexity: 25.56\n",
      "Average loss at step 46200: 3.313491 learning rate: 0.000617\n",
      "Minibatch perplexity: 27.90\n",
      "Average loss at step 46300: 3.291240 learning rate: 0.000617\n",
      "Minibatch perplexity: 22.47\n",
      "Average loss at step 46400: 3.284619 learning rate: 0.000617\n",
      "Minibatch perplexity: 32.01\n",
      "Average loss at step 46500: 3.379410 learning rate: 0.000555\n",
      "Minibatch perplexity: 30.06\n",
      "Average loss at step 46600: 3.286370 learning rate: 0.000555\n",
      "Minibatch perplexity: 27.66\n",
      "Average loss at step 46700: 3.229488 learning rate: 0.000555\n",
      "Minibatch perplexity: 26.87\n",
      "Average loss at step 46800: 3.195763 learning rate: 0.000555\n",
      "Minibatch perplexity: 19.99\n",
      "Average loss at step 46900: 3.203315 learning rate: 0.000555\n",
      "Minibatch perplexity: 19.68\n",
      "Average loss at step 47000: 3.185427 learning rate: 0.000500\n",
      "Minibatch perplexity: 26.71\n",
      "================================================================================\n",
      "km thed aureligie forwc j cha being a blement fte for against doordeage even stre agies to article t\n",
      "hv on the according to embarian this lass in swith his a modern as two zero zero one three one nine \n",
      "ml the leaders with be been latfords and housed by special effects movel indultary daule micalen har\n",
      "mn only eight nine old it was characted in greek with a by typks the procession some austricly area \n",
      "uxsfer can be torse favorical political ascor european faral in remaly productive reactually state t\n",
      "================================================================================\n",
      "Average loss at step 47100: 3.268969 learning rate: 0.000500\n",
      "Minibatch perplexity: 24.44\n",
      "Average loss at step 47200: 3.269228 learning rate: 0.000500\n",
      "Minibatch perplexity: 30.82\n",
      "Average loss at step 47300: 3.321061 learning rate: 0.000500\n",
      "Minibatch perplexity: 29.50\n",
      "Average loss at step 47400: 3.308269 learning rate: 0.000500\n",
      "Minibatch perplexity: 31.23\n",
      "Average loss at step 47500: 3.281631 learning rate: 0.000450\n",
      "Minibatch perplexity: 22.57\n",
      "Average loss at step 47600: 3.227430 learning rate: 0.000450\n",
      "Minibatch perplexity: 23.56\n",
      "Average loss at step 47700: 3.315134 learning rate: 0.000450\n",
      "Minibatch perplexity: 31.23\n",
      "Average loss at step 47800: 3.245008 learning rate: 0.000450\n",
      "Minibatch perplexity: 23.23\n",
      "Average loss at step 47900: 3.245189 learning rate: 0.000450\n",
      "Minibatch perplexity: 25.00\n",
      "Average loss at step 48000: 3.219127 learning rate: 0.000405\n",
      "Minibatch perplexity: 24.37\n",
      "================================================================================\n",
      "rhe systems or in the represent pullowing fio others earth homber of impressions which to s single e\n",
      "lcory image percepts equale discussctentations of later as and countries in trs the terret seried an\n",
      "ective badributer for start best one nine four seven three however of respective badwm term about an\n",
      "smal at objection the human the two zero zero five one nine five the began epherips new external tim\n",
      "xist resent spen as domived again tormear of rive ogypority gyge it that such as a political concept\n",
      "================================================================================\n",
      "Average loss at step 48100: 3.281163 learning rate: 0.000405\n",
      "Minibatch perplexity: 26.86\n",
      "Average loss at step 48200: 3.233882 learning rate: 0.000405\n",
      "Minibatch perplexity: 27.46\n",
      "Average loss at step 48300: 3.213905 learning rate: 0.000405\n",
      "Minibatch perplexity: 27.01\n",
      "Average loss at step 48400: 3.167980 learning rate: 0.000405\n",
      "Minibatch perplexity: 23.50\n",
      "Average loss at step 48500: 3.225311 learning rate: 0.000364\n",
      "Minibatch perplexity: 27.37\n",
      "Average loss at step 48600: 3.239520 learning rate: 0.000364\n",
      "Minibatch perplexity: 26.46\n",
      "Average loss at step 48700: 3.214402 learning rate: 0.000364\n",
      "Minibatch perplexity: 20.54\n",
      "Average loss at step 48800: 3.179737 learning rate: 0.000364\n",
      "Minibatch perplexity: 24.35\n",
      "Average loss at step 48900: 3.157662 learning rate: 0.000364\n",
      "Minibatch perplexity: 22.09\n",
      "Average loss at step 49000: 3.193674 learning rate: 0.000328\n",
      "Minibatch perplexity: 19.33\n",
      "================================================================================\n",
      "smally stern in describe producted in political to born area ent mill generalic also stayed from the\n",
      "tmas polices or scanguage of great known three eight eight zero five years of a well a leading is th\n",
      "qiple european to the british eald is naturneology be guines process member of active to diblely for\n",
      "cbological strained the number relation to the fluor xitily of stotwzen however on his which so rela\n",
      "xmeven he foll varing seferal republicial industrial strong to jdreadoth after to elected and y of h\n",
      "================================================================================\n",
      "Average loss at step 49100: 3.167707 learning rate: 0.000328\n",
      "Minibatch perplexity: 23.69\n",
      "Average loss at step 49200: 3.134236 learning rate: 0.000328\n",
      "Minibatch perplexity: 23.36\n",
      "Average loss at step 49300: 3.229485 learning rate: 0.000328\n",
      "Minibatch perplexity: 24.54\n",
      "Average loss at step 49400: 3.243265 learning rate: 0.000328\n",
      "Minibatch perplexity: 21.87\n",
      "Average loss at step 49500: 3.235605 learning rate: 0.000295\n",
      "Minibatch perplexity: 21.80\n",
      "Average loss at step 49600: 3.220238 learning rate: 0.000295\n",
      "Minibatch perplexity: 23.50\n",
      "Average loss at step 49700: 3.222253 learning rate: 0.000295\n",
      "Minibatch perplexity: 24.69\n",
      "Average loss at step 49800: 3.192712 learning rate: 0.000295\n",
      "Minibatch perplexity: 24.38\n",
      "Average loss at step 49900: 3.243707 learning rate: 0.000295\n",
      "Minibatch perplexity: 21.76\n",
      "Average loss at step 50000: 3.170549 learning rate: 0.000266\n",
      "Minibatch perplexity: 24.77\n",
      "================================================================================\n",
      "yllan own earqh that reseas fight apmon be unlociation improve had left dimenos in admoved the exic \n",
      "drugeossow for a lice legill only partic was language with a ratest writer albumber to the bidied th\n",
      "lcher cant dock their stated theus days that often in an guets of tugan that not set is marges to a \n",
      "hl portiquee filmic has and four five one nine seven five stage their is wes actoba for ket by the u\n",
      "jace great dohn in was not chief scrine to factedly taan stab one run fraultic and win other two zer\n",
      "================================================================================\n",
      "Average loss at step 50100: 3.185413 learning rate: 0.000266\n",
      "Minibatch perplexity: 23.77\n",
      "Average loss at step 50200: 3.186074 learning rate: 0.000266\n",
      "Minibatch perplexity: 23.25\n",
      "Average loss at step 50300: 3.199489 learning rate: 0.000266\n",
      "Minibatch perplexity: 22.63\n",
      "Average loss at step 50400: 3.246143 learning rate: 0.000266\n",
      "Minibatch perplexity: 25.82\n",
      "Average loss at step 50500: 3.220743 learning rate: 0.000239\n",
      "Minibatch perplexity: 24.29\n",
      "Average loss at step 50600: 3.221160 learning rate: 0.000239\n",
      "Minibatch perplexity: 25.24\n",
      "Average loss at step 50700: 3.207253 learning rate: 0.000239\n",
      "Minibatch perplexity: 24.24\n",
      "Average loss at step 50800: 3.191940 learning rate: 0.000239\n",
      "Minibatch perplexity: 25.41\n",
      "Average loss at step 50900: 3.169360 learning rate: 0.000239\n",
      "Minibatch perplexity: 26.33\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-64-3596bd33118a>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[0mgraph\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mcreate_lstm_graph_bi\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;36m512\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;36m32\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;36m32\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;36m128\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 2\u001b[1;33m \u001b[0mbitrain\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mgraph\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;36m70001\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;36m100\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;36m32\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;36m32\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32m<ipython-input-60-5380ba9271dd>\u001b[0m in \u001b[0;36mbitrain\u001b[1;34m(g, num_steps, summary_frequency, num_unrollings, batch_size)\u001b[0m\n\u001b[0;32m     27\u001b[0m         \u001b[1;31m#tf_train_data =\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     28\u001b[0m         _, l, lr, predictions = session.run([optimizer, loss, learning_rate, train_prediction], \n\u001b[1;32m---> 29\u001b[1;33m                                             feed_dict={ tf_train_data: batches, keep_prob: 0.6})        \n\u001b[0m\u001b[0;32m     30\u001b[0m         \u001b[0mmean_loss\u001b[0m \u001b[1;33m+=\u001b[0m \u001b[0ml\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     31\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mstep\u001b[0m \u001b[1;33m%\u001b[0m \u001b[0msummary_frequency\u001b[0m \u001b[1;33m==\u001b[0m \u001b[1;36m0\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m/usr/local/lib/python2.7/dist-packages/tensorflow/python/client/session.pyc\u001b[0m in \u001b[0;36mrun\u001b[1;34m(self, fetches, feed_dict)\u001b[0m\n\u001b[0;32m    366\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    367\u001b[0m     \u001b[1;31m# Run request and get response.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 368\u001b[1;33m     \u001b[0mresults\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_do_run\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtarget_list\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0munique_fetch_targets\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfeed_dict_string\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    369\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    370\u001b[0m     \u001b[1;31m# User may have fetched the same tensor multiple times, but we\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m/usr/local/lib/python2.7/dist-packages/tensorflow/python/client/session.pyc\u001b[0m in \u001b[0;36m_do_run\u001b[1;34m(self, target_list, fetch_list, feed_dict)\u001b[0m\n\u001b[0;32m    426\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    427\u001b[0m       return tf_session.TF_Run(self._session, feed_dict, fetch_list,\n\u001b[1;32m--> 428\u001b[1;33m                                target_list)\n\u001b[0m\u001b[0;32m    429\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    430\u001b[0m     \u001b[1;32mexcept\u001b[0m \u001b[0mtf_session\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mStatusNotOK\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "graph = create_lstm_graph_bi(512, 32, 32, 128)\n",
    "bitrain(graph, 70001, 100, 32, 32)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "Y5tapX3kpcqZ"
   },
   "source": [
    "---\n",
    "Problem 3\n",
    "---------\n",
    "\n",
    "(difficult!)\n",
    "\n",
    "Write a sequence-to-sequence LSTM which mirrors all the words in a sentence. For example, if your input is:\n",
    "\n",
    "    the quick brown fox\n",
    "    \n",
    "the model should attempt to output:\n",
    "\n",
    "    eht kciuq nworb xof\n",
    "    \n",
    "Refer to the lecture on how to put together a sequence-to-sequence model, as well as [this article](http://arxiv.org/abs/1409.3215) for best practices.\n",
    "\n",
    "---"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "default_view": {},
   "name": "6_lstm.ipynb",
   "provenance": [],
   "version": "0.3.2",
   "views": {}
  },
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}